1use std::{
29 collections::{BTreeMap, BTreeSet},
30 future::Future,
31 ops::ControlFlow,
32 sync::{
33 Arc,
34 atomic::{AtomicBool, AtomicU64, Ordering},
35 },
36 time::Duration,
37};
38
39use ahash::{AHashMap, AHashSet};
40use anyhow::Context;
41use async_trait::async_trait;
42use futures_util::{StreamExt, stream};
43#[cfg(test)]
44use nautilus_common::live::get_runtime;
45use nautilus_common::{
46 clients::{ExecutionClient, ExecutionReportTask},
47 enums::{LogColor, LogLevel},
48 live::runner::{get_exec_event_sender, try_get_exec_event_sender},
49 log_debug,
50 messages::{
51 ExecutionReport,
52 execution::{
53 BatchCancelOrders, BatchModifyOrders, CancelAllOrders, CancelOrder,
54 GenerateFillReports, GenerateOrderStatusReport, GenerateOrderStatusReports,
55 GeneratePositionStatusReports, ModifyOrder, QueryAccount, QueryOrder, SubmitOrder,
56 SubmitOrderList,
57 },
58 },
59 msgbus::{self, TypedHandler},
60};
61use nautilus_core::{
62 DurationNanos, UUID4, UnixNanos,
63 datetime::unix_nanos_to_iso8601,
64 params::Params,
65 string::secret::SecretString,
66 time::{AtomicTime, get_atomic_clock_realtime},
67};
68use nautilus_execution::reconciliation::{
69 inferred_reconciliation_trade_ids, is_inferred_reconciliation_trade_id_format,
70};
71use nautilus_live::{
72 ExecutionClientCore, ExecutionEventEmitter, SocketControlFactory,
73 book::DEFAULT_BOOK_SNAPSHOT_TIMEOUT_SECS,
74 execution::failure::CommandFailure,
75 task::{TaskGroup, TaskGroupGuard, TaskJoinOutcome, TaskSlot, TaskSpawner, finish_task},
76};
77use nautilus_model::{
78 accounts::AccountAny,
79 enums::{
80 AccountType, ContingencyType, OmsType, OrderSide, OrderStatus, OrderType, PositionSide,
81 TimeInForce,
82 },
83 events::{OrderAccepted, OrderDeniedReason, OrderEventAny, OrderFilled, OrderTriggered},
84 identifiers::{
85 AccountId, ClientId, ClientOrderId, InstrumentId, StrategyId, TradeId, TraderId, Venue,
86 VenueOrderId,
87 },
88 instruments::{Instrument, InstrumentAny},
89 orders::{LIMIT_ORDER_TYPES, Order, OrderAny, STOP_ORDER_TYPES, TRIGGERABLE_ORDER_TYPES},
90 reports::{ExecutionMassStatus, FillReport, OrderStatusReport, PositionStatusReport},
91 types::{AccountBalance, MarginBalance, Quantity},
92};
93use nautilus_network::error::SendError;
94use parking_lot::Mutex;
95use rust_decimal::Decimal;
96use serde::Deserialize;
97use tokio_util::sync::CancellationToken;
98use zeroize::Zeroizing;
99
100use crate::{
101 common::{
102 consts::{
103 DISCONNECT_TIMEOUT, LIGHTER_ERROR_CODE_INVALID_NONCE, LIGHTER_MAX_BATCH_TX,
104 SCHEDULED_CANCEL_MAX_DELAY_SECS, SCHEDULED_CANCEL_MIN_DELAY_SECS,
105 SCHEDULED_CANCEL_REQUEST_TIMEOUT,
106 },
107 credential::{Credential, scrub_auth},
108 deployment,
109 enums::{
110 LighterAccountTier, LighterCancelAllTimeInForce, LighterGroupingType,
111 LighterOrderStatus, LighterPositionMarginMode, LighterProductType,
112 LighterSelfTradeBehaviorMode, LighterSelfTradeEqualityMode, LighterTxStatus,
113 LighterTxType,
114 },
115 parse::{parse_price_from_ticks, parse_quantity_from_ticks, quantity_from_decimal},
116 rate_limit::{LighterTxRateLimiter, await_tx_quota, build_tx_rate_limiter, resolve_quota},
117 symbol::{MarketRegistry, product_type_from_instrument_id},
118 },
119 config::{LighterExecutionClientConfig, validate_scheduled_cancel_timeout},
120 http::{
121 client::{LIGHTER_REST_PAGE_SIZE, LighterHttpClient, LighterRawHttpClient},
122 error::LighterHttpError,
123 models::{
124 LighterAccountDetail, LighterOrder, LighterSendTxRequest, LighterTrade, LighterTx,
125 },
126 query::{
127 LighterAccountActiveOrdersQuery, LighterAccountInactiveOrdersQuery,
128 LighterSortDirection, LighterTradeSortBy, LighterTradesQuery,
129 },
130 },
131 signing::{
132 auth_token::{build_auth_token_for, fresh_k},
133 nonce::NonceError,
134 tx::{
135 ApproveIntegratorTxInfo, CancelAllOrdersTxInfo, CancelOrderTxInfo,
136 CreateGroupedOrdersTxInfo, CreateOrderTxInfo, L2TxAttributes, ModifyOrderTxInfo,
137 OrderInfo, TxContext, TxInfoJson, UpdateLeverageTxInfo, sign_tx,
138 },
139 },
140 websocket::{
141 LighterWsError, USER_STREAMS_ENDPOINT,
142 client::{LighterWebSocketClient, RetainedTaskSlot, TaskRetentionGuard},
143 dispatch::{
144 DeferredOrderEvents, LIGHTER_INSTRUMENT_CACHE, MAX_RECONCILIATION_PAGES, OrderIdentity,
145 OrderReportLookup, OrderShapeSnapshot, PendingOrderAction, PendingSendTx,
146 PendingSendTxKind, PendingTerminal, TradeDedupSource, WsDispatchState,
147 cache_instruments_for_reports, collect_order, collect_order_while_pending,
148 derive_market_order_price_ticks, evict_terminal_mappings, fills_for_order,
149 lookup_create_order_status_report, nautilus_to_lighter_order_type,
150 nautilus_to_lighter_tif, order_expiry_for, parse_http_order_to_report, price_to_ticks,
151 quantity_to_ticks,
152 },
153 messages::{
154 AccountStream, ExecutionReport as LighterExecutionReport, LighterWsChannel,
155 NautilusWsMessage, SendTxRejectionSource, TX_BATCH_ID_PREFIX,
156 },
157 parse::{
158 LighterCommissionError, OpenFrameContext, ParsedOrderEvent, is_order_triggered,
159 lighter_order_shape, parse_lighter_order_event, parse_lighter_order_filled,
160 parse_lighter_order_updated, parse_lighter_trade_id, parse_ws_fill_report,
161 parse_ws_order_status_report,
162 },
163 },
164};
165
166const DEFAULT_TX_EXPIRY_MS: i64 = 5 * 60 * 1_000;
169
170const ACKED_ORDER_LOOKUP_DELAY: Duration = Duration::from_secs(2);
172const ACKED_ORDER_PROBE_ATTEMPTS: usize = 3;
173const TERMINAL_FILL_DELAY: Duration = Duration::from_secs(2);
174const TERMINAL_RECOVERY_TIMEOUT: Duration = Duration::from_secs(2);
175const TERMINAL_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(7);
176const ORDER_EVENTS_PATTERN: &str = "events.order.*";
177const BATCH_ACK_TIMEOUT: Duration = Duration::from_secs(10);
178const ACCOUNT_CANCEL_CONFIRM_TIMEOUT: Duration = Duration::from_secs(10);
179
180const STRATEGY_REASON_MAX_CHARS: usize = 512;
181
182const AUTH_TOKEN_REFRESH_INTERVAL: std::time::Duration =
187 std::time::Duration::from_secs(6 * 60 * 60);
188
189const _: () = assert!(
191 AUTH_TOKEN_REFRESH_INTERVAL.as_secs()
192 < crate::signing::auth_token::DEFAULT_AUTH_TOKEN_TTL_SECS as u64,
193 "AUTH_TOKEN_REFRESH_INTERVAL must stay below DEFAULT_AUTH_TOKEN_TTL_SECS",
194);
195
196const AUTH_TOKEN_REFRESH_RETRY_WINDOW: Duration = Duration::from_secs(60 * 60);
199const AUTH_TOKEN_REFRESH_RETRY_INITIAL_DELAY: Duration = Duration::from_secs(30);
200
201const AUTH_TOKEN_REFRESH_RETRY_MAX_DELAY: Duration = Duration::from_secs(5 * 60);
203const AUTH_TOKEN_REFRESH_BACKOFF: AuthTokenRefreshBackoff = AuthTokenRefreshBackoff {
204 initial_delay: AUTH_TOKEN_REFRESH_RETRY_INITIAL_DELAY,
205 max_delay: AUTH_TOKEN_REFRESH_RETRY_MAX_DELAY,
206 window: AUTH_TOKEN_REFRESH_RETRY_WINDOW,
207};
208const NONCE_REFRESH_RETRY_INITIAL_DELAY: Duration = Duration::from_secs(1);
209const NONCE_REFRESH_RETRY_MAX_DELAY: Duration = Duration::from_secs(30);
210const ACCOUNT_SNAPSHOT_TIMEOUT: Duration = Duration::from_secs(10);
213const REFERRAL_ATTRIBUTION_TIMEOUT: Duration = Duration::from_secs(10);
214
215const SENDTX_BARE_ERROR_WINDOW_MS: u64 = 1_000;
220const INTEGRATOR_AUTO_APPROVAL_MAX_TTL_MS: i64 = 5 * 365 * 24 * 60 * 60 * 1_000;
221const INTEGRATOR_AUTO_APPROVAL_MAX_FEE_TICK: u32 = 0;
222const NONCE_CONNECTION_EPOCH_UNAVAILABLE: u64 = u64::MAX;
223
224#[derive(Debug)]
225pub struct LighterExecutionClient {
226 core: ExecutionClientCore,
227 clock: &'static AtomicTime,
228 config: LighterExecutionClientConfig,
229 emitter: ExecutionEventEmitter,
230 credential: Option<Credential>,
231 http_client: LighterHttpClient,
232 ws_client: LighterWebSocketClient,
233 tx_rate_limiter: Arc<LighterTxRateLimiter>,
234 tx_send_sequencer: TxSendSequencer,
235 nonce_submission_gate: Arc<tokio::sync::RwLock<()>>,
236 nonce_ready_connection_epoch: Arc<AtomicU64>,
237 registry: Arc<MarketRegistry>,
238 socket_factory: SocketControlFactory,
239 pending_tasks: TaskGroup,
240 ws_stream_handle: TaskSlot<()>,
241 terminal_recovery_handle: TaskSlot<()>,
242 terminal_producers_stopped: CancellationToken,
243 fills_applied: Option<AppliedFillSubscription>,
244 ws_disconnect_handle: TaskSlot<Result<(), LighterWsError>>,
245 ws_handler_retained: Arc<RetainedTaskSlot>,
246 auth_refresh_handle: TaskSlot<()>,
247 scheduled_cancel_handle: TaskSlot<()>,
248 scheduled_cancel_notify: Arc<tokio::sync::Notify>,
249 shutdown_errors: Vec<String>,
250 cancellation_token: CancellationToken,
251 dispatch: WsDispatchState,
252 nonce_recovery_inflight: Arc<AtomicBool>,
253 auth_refresh_notify: Arc<tokio::sync::Notify>,
254}
255
256impl LighterExecutionClient {
257 fn log_report_receipt(count: usize, report_type: &str, log_level: LogLevel) {
258 let plural = if count == 1 { "" } else { "s" };
259 let message = format!("Received {count} {report_type}{plural}");
260
261 match log_level {
262 LogLevel::Off => {}
263 LogLevel::Trace => log::trace!("{message}"),
264 LogLevel::Debug => log::debug!("{message}"),
265 LogLevel::Info => log::info!("{message}"),
266 LogLevel::Warning => log::warn!("{message}"),
267 LogLevel::Error => log::error!("{message}"),
268 }
269 }
270
271 pub fn new(
284 core: ExecutionClientCore,
285 config: LighterExecutionClientConfig,
286 ) -> anyhow::Result<Self> {
287 validate_scheduled_cancel_timeout(config.scheduled_cancel_timeout_secs)?;
288 anyhow::ensure!(
289 core.venue == config.resolved_venue(),
290 "Lighter execution core venue {} does not match configured venue {}",
291 core.venue,
292 config.resolved_venue(),
293 );
294
295 anyhow::ensure!(
296 config.account_id.get_issuer() == core.venue,
297 "Lighter account ID issuer {} does not match configured venue {}",
298 config.account_id.get_issuer(),
299 core.venue,
300 );
301
302 let credential = Credential::resolve_for_deployment(
303 config.private_key.clone().map(SecretString::into_inner),
304 config.account_index,
305 config.api_key_index,
306 config.deployment,
307 config.environment,
308 )
309 .context("failed to resolve Lighter credentials")?;
310
311 let registry = Arc::new(MarketRegistry::new_with_venue_and_settlement_currency(
312 core.venue,
313 config.settlement_currency(),
314 ));
315 let socket_factory = SocketControlFactory::new(core.client_id, Some(core.venue));
316
317 let tx_rate_limiter = build_tx_rate_limiter(config.sendtx_quota_per_min);
320
321 let raw_http = LighterRawHttpClient::new_with_quotas(
322 config.environment,
323 Some(config.http_url()),
324 config.http_timeout_secs,
325 config
326 .proxy_url
327 .as_ref()
328 .map(|value| value.expose_secret().to_owned()),
329 resolve_quota(config.rest_quota_per_min),
330 Some(Arc::clone(&tx_rate_limiter)),
331 )
332 .context("failed to construct Lighter raw HTTP client")?;
333
334 let http_client =
335 LighterHttpClient::from_raw_with_registry(raw_http, Arc::clone(®istry));
336
337 let ws_client = Self::create_ws_client(&config, Arc::clone(®istry), &socket_factory);
338
339 let clock = get_atomic_clock_realtime();
340
341 let mut emitter = ExecutionEventEmitter::new(
342 clock,
343 core.trader_id,
344 core.account_id,
345 AccountType::Margin,
346 None,
347 );
348
349 if let Some(sender) = try_get_exec_event_sender() {
350 emitter.set_sender(sender);
351 }
352
353 let pending_tasks = TaskGroup::new();
354
355 Ok(Self {
356 core,
357 clock,
358 config,
359 emitter,
360 credential,
361 http_client,
362 ws_client,
363 tx_rate_limiter,
364 tx_send_sequencer: TxSendSequencer::new(),
365 nonce_submission_gate: Arc::new(tokio::sync::RwLock::new(())),
366 nonce_ready_connection_epoch: Arc::new(AtomicU64::new(0)),
367 registry,
368 socket_factory,
369 cancellation_token: pending_tasks.cancellation_token(),
370 pending_tasks,
371 ws_stream_handle: TaskSlot::new(),
372 terminal_recovery_handle: TaskSlot::new(),
373 terminal_producers_stopped: CancellationToken::new(),
374 fills_applied: None,
375 ws_disconnect_handle: TaskSlot::new(),
376 ws_handler_retained: Arc::new(RetainedTaskSlot::new()),
377 auth_refresh_handle: TaskSlot::new(),
378 scheduled_cancel_handle: TaskSlot::new(),
379 scheduled_cancel_notify: Arc::new(tokio::sync::Notify::new()),
380 shutdown_errors: Vec::new(),
381 dispatch: WsDispatchState::new(),
382 nonce_recovery_inflight: Arc::new(AtomicBool::new(false)),
383 auth_refresh_notify: Arc::new(tokio::sync::Notify::new()),
384 })
385 }
386
387 #[must_use]
389 pub fn config(&self) -> &LighterExecutionClientConfig {
390 &self.config
391 }
392
393 #[must_use]
395 pub fn has_credentials(&self) -> bool {
396 self.credential.is_some()
397 }
398
399 #[must_use]
402 pub fn pending_tasks_all_finished(&self) -> bool {
403 self.pending_tasks.all_finished()
404 }
405
406 pub async fn cancel_all_orders_account(&self) -> anyhow::Result<String> {
423 self.send_cancel_all(LighterCancelAllTimeInForce::Immediate, 0)
424 .await
425 }
426
427 pub async fn schedule_cancel(&self, time: Option<u64>) -> anyhow::Result<String> {
447 let (time_in_force, scheduled_time_ms) = match time {
448 Some(time) => {
449 let now = self.clock.get_time_ms();
450 let delay = time
451 .checked_sub(now)
452 .context("scheduled cancellation is in the past")?;
453 anyhow::ensure!(
454 (SCHEDULED_CANCEL_MIN_DELAY_SECS * 1_000
455 ..=SCHEDULED_CANCEL_MAX_DELAY_SECS * 1_000)
456 .contains(&delay),
457 "scheduled cancellation must be between five minutes and fifteen days ahead",
458 );
459 let time = i64::try_from(time)
460 .context("scheduled cancellation timestamp overflows milliseconds")?;
461 (LighterCancelAllTimeInForce::Scheduled, time)
462 }
463 None => (LighterCancelAllTimeInForce::AbortScheduled, 0),
464 };
465
466 self.send_cancel_all(time_in_force, scheduled_time_ms).await
467 }
468
469 async fn send_cancel_all(
470 &self,
471 time_in_force: LighterCancelAllTimeInForce,
472 scheduled_time_ms: i64,
473 ) -> anyhow::Result<String> {
474 let credential = self
475 .credential
476 .as_ref()
477 .context("Lighter account cancellation requires credentials")?;
478 self.fanout_dispatch_context(credential)
479 .context("failed to spawn Lighter account cancellation")?
480 .send_cancel_all(
481 time_in_force,
482 scheduled_time_ms,
483 false,
484 self.cancellation_token.clone(),
485 )
486 .await
487 }
488
489 async fn start_scheduled_cancel(&mut self) -> anyhow::Result<()> {
490 let Some(timeout_secs) = self.config.scheduled_cancel_timeout_secs else {
491 return Ok(());
492 };
493
494 let credential = self
495 .credential
496 .as_ref()
497 .context("scheduled cancellation requires credentials")?;
498 let context = self.fanout_dispatch_context(credential)?;
499 context
500 .refresh_scheduled_cancel(timeout_secs)
501 .await
502 .context("failed to arm Lighter scheduled cancellation")?;
503 let cancellation = self.cancellation_token.clone();
504 let notify = Arc::clone(&self.scheduled_cancel_notify);
505 let refresh_interval =
506 Duration::from_secs(((timeout_secs - SCHEDULED_CANCEL_MIN_DELAY_SECS) / 3).min(60));
507
508 self.scheduled_cancel_handle.spawn(async move {
509 let mut interval = tokio::time::interval_at(tokio::time::Instant::now() + refresh_interval, refresh_interval);
510 interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
511
512 loop {
513 tokio::select! {
514 biased;
515 () = cancellation.cancelled() => break,
516 () = notify.notified() => {},
517 _ = interval.tick() => {},
518 }
519
520 if !context.ws_client.is_active()
521 || context.nonce_ready_connection_epoch.load(Ordering::Acquire) != context.ws_client.connection_epoch() {
522 continue;
523 }
524
525 tokio::select! {
526 biased;
527 () = cancellation.cancelled() => break,
528 result = context.refresh_scheduled_cancel(timeout_secs) => {
529 if let Err(e) = result {
530 log::error!("Scheduled cancellation refresh failed; the last deadline may expire: {e:#}");
531 }
532 }
533 }
534 }
535 })?;
536
537 log::info!("Scheduled cancellation armed: timeout_secs={timeout_secs}");
538 Ok(())
539 }
540
541 fn create_ws_client(
542 config: &LighterExecutionClientConfig,
543 registry: Arc<MarketRegistry>,
544 socket_factory: &SocketControlFactory,
545 ) -> LighterWebSocketClient {
546 let ws_client = LighterWebSocketClient::new(
547 Some(config.ws_url()),
548 config.environment,
549 registry,
550 config.transport_backend,
551 config.ws_timeout_secs,
552 Duration::from_secs(DEFAULT_BOOK_SNAPSHOT_TIMEOUT_SECS),
554 config
555 .proxy_url
556 .as_ref()
557 .map(|value| value.expose_secret().to_owned()),
558 );
559
560 ws_client.with_socket_control(socket_factory.control(USER_STREAMS_ENDPOINT))
561 }
562
563 fn take_ws_client(&mut self) -> LighterWebSocketClient {
564 let cached_instruments = self.ws_client.instruments_cache();
565 let ws_cache = cached_instruments
566 .iter()
567 .map(|entry| (*entry.key(), entry.value().clone()))
568 .collect();
569 let replacement = Self::create_ws_client(
570 &self.config,
571 Arc::clone(&self.registry),
572 &self.socket_factory,
573 );
574 replacement.cache_instruments(ws_cache);
575
576 std::mem::replace(&mut self.ws_client, replacement)
577 }
578
579 fn spawn_task<F>(&self, description: &'static str, fut: F)
580 where
581 F: std::future::Future<Output = anyhow::Result<()>> + Send + 'static,
582 {
583 let future = async move {
584 if let Err(e) = fut.await {
585 log::warn!("{description} failed: {e:?}");
586 }
587 };
588
589 if let Err(e) = self.pending_tasks.spawn(future) {
590 log::warn!("Skipping Lighter {description} after shutdown began: {e}");
591 }
592 }
593
594 fn subscribe_applied_fills(&mut self) {
595 if self.fills_applied.is_some() {
596 return;
597 }
598
599 let core = self.core.clone();
600 let dispatch = self.dispatch.clone();
601 let emitter = self.emitter.clone();
602
603 let handler = TypedHandler::from(move |event: &OrderEventAny| {
604 let OrderEventAny::Filled(fill) = event else {
605 return;
606 };
607
608 if fill.account_id != core.account_id || fill.trader_id != core.trader_id {
609 return;
610 }
611
612 let Some(identity) = dispatch.order_identity(&fill.client_order_id) else {
613 return;
614 };
615
616 if identity.strategy_id != fill.strategy_id
617 || identity.instrument_id != fill.instrument_id
618 {
619 return;
620 }
621
622 let cache = core.cache();
623
624 let Some(order) = cache.order(&fill.client_order_id) else {
625 return;
626 };
627
628 if !order.is_duplicate_fill(fill) {
629 return;
630 }
631
632 let events = dispatch.order_events.lock();
633
634 if is_inferred_reconciliation_trade_id_format(&fill.trade_id) {
635 identity.sync_fills(&order);
636 }
637
638 if identity.has_inferred_fills() {
639 return;
640 }
641
642 identity.publish_fill(fill);
643 identity.publication_pending(Some(&order));
644 dispatch.mark_trade_reconciled(fill.trade_id);
645 release_terminal(fill.client_order_id, false, &dispatch, &emitter, &events);
646 });
647
648 msgbus::subscribe_order_events(ORDER_EVENTS_PATTERN.into(), handler.clone(), Some(10));
649 self.fills_applied = Some(AppliedFillSubscription(handler));
650 }
651
652 fn begin_session_shutdown(&mut self) {
653 self.pending_tasks.begin_shutdown();
654 self.cancellation_token.cancel();
655 self.ws_client.begin_shutdown();
656 self.core.set_disconnected();
657
658 if self.ws_disconnect_handle.is_none()
659 && (self.ws_client.is_active() || self.ws_stream_handle.is_some())
660 {
661 let ws_client = self.take_ws_client();
662 let retained = Arc::clone(&self.ws_handler_retained);
663
664 if let Err(e) = self
665 .ws_disconnect_handle
666 .spawn(ws_client.disconnect_with_task_retention(retained))
667 {
668 log::error!("Failed to start Lighter WebSocket disconnect task: {e}");
669 }
670 }
671 }
672
673 fn session_tasks_finished(&self) -> bool {
674 self.ws_stream_handle.is_none()
675 && self.terminal_recovery_handle.is_none()
676 && self.ws_disconnect_handle.is_none()
677 && self.ws_handler_retained.is_empty()
678 && self.auth_refresh_handle.is_none()
679 && self.scheduled_cancel_handle.is_none()
680 && self.pending_tasks.is_empty()
681 && self.shutdown_errors.is_empty()
682 }
683
684 async fn finish_session_shutdown(&mut self) -> anyhow::Result<()> {
685 self.pending_tasks.begin_shutdown();
686
687 Self::finish_disconnect_task(
688 &mut self.ws_disconnect_handle,
689 "WebSocket disconnect",
690 &mut self.shutdown_errors,
691 )
692 .await;
693
694 if let Err(e) = self.ws_handler_retained.finish().await {
695 self.shutdown_errors.push(e.to_string());
696 }
697 Self::finish_owned_task(
698 &mut self.ws_stream_handle,
699 "execution consumer",
700 &mut self.shutdown_errors,
701 )
702 .await;
703
704 Self::finish_owned_task(
705 &mut self.auth_refresh_handle,
706 "auth-token refresh",
707 &mut self.shutdown_errors,
708 )
709 .await;
710
711 if let Err(e) = self
712 .pending_tasks
713 .finish_shutdown(Duration::from_secs(1), DISCONNECT_TIMEOUT)
714 .await
715 {
716 self.shutdown_errors
717 .push(format!("client-owned tasks failed: {e}"));
718 }
719
720 Self::finish_owned_task(
721 &mut self.scheduled_cancel_handle,
722 "scheduled cancellation refresh",
723 &mut self.shutdown_errors,
724 )
725 .await;
726
727 if self.ws_disconnect_handle.is_none()
728 && self.ws_handler_retained.is_empty()
729 && self.ws_stream_handle.is_none()
730 && self.pending_tasks.is_empty()
731 {
732 self.terminal_producers_stopped.cancel();
733
734 if let Some(outcome) = finish_task(
735 &mut self.terminal_recovery_handle,
736 TERMINAL_SHUTDOWN_TIMEOUT + DISCONNECT_TIMEOUT,
737 DISCONNECT_TIMEOUT,
738 )
739 .await
740 && !matches!(outcome, TaskJoinOutcome::Completed(()))
741 {
742 self.shutdown_errors
743 .push(format!("terminal recovery task failed: {outcome:?}"));
744 }
745 }
746
747 if self.session_tasks_finished() {
748 self.fills_applied = None;
749 }
750
751 let stale = self.dispatch.take_pending_sendtx();
752 warn_pending_sendtx_unknown(&stale, "session shutdown");
753 self.nonce_recovery_inflight.store(false, Ordering::Release);
754
755 if !self.shutdown_errors.is_empty() {
756 let errors = std::mem::take(&mut self.shutdown_errors);
757 anyhow::bail!("Failed to terminate Lighter tasks: {}", errors.join("; "));
758 }
759 Ok(())
760 }
761
762 async fn finish_owned_task(
763 slot: &mut TaskSlot<()>,
764 description: &str,
765 errors: &mut Vec<String>,
766 ) {
767 let Some(outcome) = finish_task(slot, DISCONNECT_TIMEOUT, DISCONNECT_TIMEOUT).await else {
768 return;
769 };
770
771 match outcome {
772 TaskJoinOutcome::Completed(()) => {
773 log::debug!("Lighter {description} task completed");
774 }
775 TaskJoinOutcome::Aborted => {
776 log::debug!("Lighter {description} task cancelled");
777 }
778 TaskJoinOutcome::Failed(e) => {
779 errors.push(format!("{description} task failed: {e}"));
780 }
781 TaskJoinOutcome::Incomplete => {
782 errors.push(format!("{description} task did not stop after abort"));
783 }
784 }
785 }
786
787 async fn finish_disconnect_task(
788 slot: &mut TaskSlot<Result<(), LighterWsError>>,
789 description: &str,
790 errors: &mut Vec<String>,
791 ) {
792 let Some(outcome) = finish_task(slot, DISCONNECT_TIMEOUT, DISCONNECT_TIMEOUT).await else {
793 return;
794 };
795
796 match outcome {
797 TaskJoinOutcome::Completed(Ok(())) | TaskJoinOutcome::Aborted => {}
798 TaskJoinOutcome::Completed(Err(e)) => {
799 errors.push(format!("{description} failed: {e}"));
800 }
801 TaskJoinOutcome::Failed(e) => {
802 errors.push(format!("{description} task failed: {e}"));
803 }
804 TaskJoinOutcome::Incomplete => {
805 errors.push(format!("{description} task did not stop after abort"));
806 }
807 }
808 }
809
810 async fn ensure_instruments_initialized_async(&self) -> anyhow::Result<()> {
811 if self.core.instruments_initialized() {
812 return Ok(());
813 }
814
815 let instruments = self
816 .http_client
817 .request_instruments()
818 .await
819 .context("failed to request Lighter instruments")?;
820
821 let ws_cache: Vec<(i64, InstrumentAny)> = instruments
822 .iter()
823 .filter_map(|instrument| {
824 self.registry
825 .market_index(&instrument.id())
826 .map(|market_index| (market_index, instrument.clone()))
827 })
828 .collect();
829 self.ws_client.cache_instruments(ws_cache);
830 cache_instruments_for_reports(&instruments);
831
832 log::debug!(
833 "Bootstrapped {} Lighter instruments ({} registry entries)",
834 instruments.len(),
835 self.registry.len(),
836 );
837
838 self.core.set_instruments_initialized();
839 Ok(())
840 }
841
842 async fn await_account_streams_ready(&self, timeout_secs: f64) -> anyhow::Result<()> {
843 let timeout = Duration::from_secs_f64(timeout_secs);
844 self.dispatch.account_streams_ready.await_all(timeout).await
845 }
846
847 async fn refresh_nonce(&self) -> anyhow::Result<()> {
848 let Some(credential) = &self.credential else {
849 return Ok(());
850 };
851 self.nonce_ready_connection_epoch
852 .store(NONCE_CONNECTION_EPOCH_UNAVAILABLE, Ordering::Release);
853 let _refresh_guard = self.nonce_submission_gate.write().await;
854
855 let response = self
856 .http_client
857 .get_next_nonce(credential.account_index(), credential.api_key_index())
858 .await
859 .context("failed to fetch Lighter nextNonce")?;
860
861 self.dispatch.nonce_manager.refresh(
862 credential.account_index(),
863 credential.api_key_index(),
864 response.nonce,
865 );
866
867 self.nonce_recovery_inflight.store(false, Ordering::Release);
869
870 log::debug!(
871 "Refreshed Lighter nonce baseline: account_index={}, api_key_index={}, next_nonce={}",
872 credential.account_index(),
873 credential.api_key_index(),
874 response.nonce,
875 );
876 Ok(())
877 }
878
879 async fn sync_nonce_from_venue(&self) -> anyhow::Result<()> {
880 let Some(credential) = &self.credential else {
881 return Ok(());
882 };
883
884 let _refresh_guard = self.nonce_submission_gate.write().await;
885 let response = self
886 .http_client
887 .get_next_nonce(credential.account_index(), credential.api_key_index())
888 .await
889 .context("failed to fetch Lighter nextNonce")?;
890 self.dispatch.nonce_manager.sync_from_venue(
891 credential.account_index(),
892 credential.api_key_index(),
893 response.nonce,
894 )?;
895 Ok(())
896 }
897
898 async fn fetch_account_detail(&self) -> Option<LighterAccountDetail> {
899 let Some(credential) = &self.credential else {
900 return None;
901 };
902 let account_index = credential.account_index();
903
904 match tokio::time::timeout(
905 ACCOUNT_SNAPSHOT_TIMEOUT,
906 self.http_client.get_account_detail(account_index),
907 )
908 .await
909 {
910 Ok(Ok(detail)) => Some(detail),
911 Ok(Err(e)) => {
912 log::warn!(
913 "Failed to fetch Lighter account detail for account_index={account_index}; \
914 continuing startup without account metadata: {e}"
915 );
916 None
917 }
918 Err(_) => {
919 log::warn!(
920 "Lighter account detail timed out after {}s for account_index={account_index}; \
921 continuing startup without account metadata",
922 ACCOUNT_SNAPSHOT_TIMEOUT.as_secs()
923 );
924 None
925 }
926 }
927 }
928
929 fn log_account_tier(&self, detail: &LighterAccountDetail) {
934 let account_index = detail.account_index;
935 let code = detail.account_type;
936 let tier = LighterAccountTier::from_code(code);
937 let standard_rest = LighterAccountTier::Standard
938 .documented_rest_quota_per_min()
939 .unwrap_or(60);
940 let (active_rest, cross_check) =
941 tier_quota_report(tier, self.config.rest_quota_per_min, standard_rest);
942
943 log_debug!(
944 "Lighter execution account {account_index} reported tier {tier} \
945 (account_type={code}); active REST quota {active_rest} req/min",
946 color = LogColor::Blue
947 );
948
949 match cross_check {
950 Some(TierCrossCheck::AboveTier { documented }) => {
951 log::warn!(
952 "Configured Lighter rest_quota_per_min={active_rest} exceeds the {tier} tier \
953 limit of {documented} req/min; the venue may reject requests unless the \
954 caller IP is registered for the higher limit"
955 );
956 }
957 Some(TierCrossCheck::RaiseHint { documented }) => {
958 log_debug!(
959 "Lighter {tier} tier permits up to {documented} REST req/min; set \
960 rest_quota_per_min (and register the caller IP with Lighter) to use it",
961 color = LogColor::Blue
962 );
963 }
964 None => {}
965 }
966 }
967
968 fn integrator_account_index(&self) -> Option<u64> {
969 deployment::integrator_account_index(self.config.deployment, self.config.environment)
970 }
971
972 async fn apply_referral_attribution(
973 &self,
974 detail: &LighterAccountDetail,
975 ) -> anyhow::Result<()> {
976 let Some(referral_code) =
977 deployment::referral_code(self.config.deployment, self.config.environment)
978 else {
979 return Ok(());
980 };
981
982 let Some(credential) = &self.credential else {
983 return Ok(());
984 };
985
986 anyhow::ensure!(
987 !detail.l1_address.trim().is_empty(),
988 "Lighter account detail returned an empty L1 address"
989 );
990
991 let auth_token = build_auth_token_for(credential)
992 .context("failed to mint Lighter auth token for referral attribution")?;
993 let referral_code = Zeroizing::new(referral_code.to_string());
994
995 self.http_client
996 .use_referral(&detail.l1_address, referral_code.as_str(), &auth_token)
997 .await
998 .context("failed to apply Lighter referral code")?;
999
1000 Ok(())
1001 }
1002
1003 async fn is_maker_only_api_key(&self, credential: &Credential) -> anyhow::Result<bool> {
1007 let auth_token = build_auth_token_for(credential)
1008 .context("failed to mint Lighter auth token for maker-only check")?;
1009 let response = self
1010 .http_client
1011 .get_maker_only_api_keys(credential.account_index(), auth_token)
1012 .await
1013 .context("failed to query getMakerOnlyApiKeys")?;
1014 let api_key_index = i64::from(credential.api_key_index());
1015 Ok(response.api_key_indexes.contains(&api_key_index))
1016 }
1017
1018 async fn submit_integrator_auto_approval(&self) -> anyhow::Result<()> {
1019 let Some(integrator_account_index) = self.integrator_account_index() else {
1020 return Ok(());
1021 };
1022
1023 let Some(credential) = &self.credential else {
1024 return Ok(());
1025 };
1026
1027 let mut maker_only_check_failed = false;
1028
1029 match self.is_maker_only_api_key(credential).await {
1030 Ok(true) => {
1031 log::warn!(
1032 "Skipping Lighter integrator auto-approval: api_key_index={} is maker-only; \
1033 ensure the account has been approved by a non-maker-only key",
1034 credential.api_key_index(),
1035 );
1036 return Ok(());
1037 }
1038 Ok(false) => {}
1039 Err(e) => {
1040 maker_only_check_failed = true;
1041
1042 log::debug!(
1043 "Unable to determine whether the Lighter API key is maker-only; proceeding \
1044 with integrator auto-approval: {e:?}"
1045 );
1046 }
1047 }
1048
1049 let mut approval =
1050 self.prepare_integrator_auto_approval(credential, integrator_account_index)?;
1051
1052 let request = LighterSendTxRequest::new(
1053 LighterTxType::ApproveIntegrator as u8,
1054 approval.tx_info.clone(),
1055 );
1056
1057 approval.send_reservation.wait_for_turn().await;
1058
1059 let response = match self.http_client.send_tx(&request).await {
1060 Ok(response) => response,
1061 Err(e) => {
1062 if lighter_http_error_is_definite_api_rejection(&e) {
1063 let _ = self.dispatch.nonce_manager.ack_failure_if_latest(
1064 credential.account_index(),
1065 approval.api_key_index,
1066 approval.nonce,
1067 );
1068 }
1069 approval.send_reservation.release();
1070 let hint = if maker_only_check_failed {
1071 " (maker-only pre-flight check failed earlier; venue may reject with 62007 \
1072 if this key is maker-only)"
1073 } else {
1074 ""
1075 };
1076 return Err(anyhow::Error::new(e).context(format!(
1077 "failed to submit Lighter integrator approval nonce={} api_key_index={}{hint}",
1078 approval.nonce, approval.api_key_index,
1079 )));
1080 }
1081 };
1082
1083 let _ = self.dispatch.nonce_manager.ack_success(
1084 credential.account_index(),
1085 approval.api_key_index,
1086 approval.nonce,
1087 );
1088 approval.send_reservation.release();
1089
1090 log::debug!(
1091 "Submitted Lighter integrator approval: integrator={}, nonce={}, \
1092 api_key_index={}, approval_expiry={}, tx_hash={}",
1093 integrator_account_index,
1094 approval.nonce,
1095 approval.api_key_index,
1096 approval.approval_expiry,
1097 response.tx_hash,
1098 );
1099 Ok(())
1100 }
1101
1102 fn prepare_integrator_auto_approval(
1103 &self,
1104 credential: &Credential,
1105 integrator_account_index: u64,
1106 ) -> anyhow::Result<PreparedIntegratorApproval> {
1107 let ReservedTxContext {
1108 context,
1109 send_reservation,
1110 } = self.build_tx_context(credential)?;
1111
1112 let now_ms = (self.clock.get_time_ns().as_u64() as i64) / 1_000_000;
1113 let approval_expiry = now_ms.saturating_add(INTEGRATOR_AUTO_APPROVAL_MAX_TTL_MS);
1114 let nonce = context.nonce;
1115 let api_key_index = context.api_key_index;
1116
1117 let tx = ApproveIntegratorTxInfo {
1118 context,
1119 integrator_account_index: integrator_account_index as i64,
1120 max_perps_taker_fee: INTEGRATOR_AUTO_APPROVAL_MAX_FEE_TICK,
1121 max_perps_maker_fee: INTEGRATOR_AUTO_APPROVAL_MAX_FEE_TICK,
1122 max_spot_taker_fee: INTEGRATOR_AUTO_APPROVAL_MAX_FEE_TICK,
1123 max_spot_maker_fee: INTEGRATOR_AUTO_APPROVAL_MAX_FEE_TICK,
1124 approval_expiry,
1125 skip_nonce: 0,
1126 };
1127
1128 let signed = sign_tx(
1129 &tx,
1130 self.config.chain_id(),
1131 &credential.private_key()?,
1132 fresh_k(),
1133 )?;
1134
1135 let tx_info = TxInfoJson::approve_integrator(&tx, &signed, "");
1136
1137 Ok(PreparedIntegratorApproval {
1138 tx_info,
1139 nonce,
1140 api_key_index,
1141 approval_expiry,
1142 send_reservation,
1143 })
1144 }
1145
1146 async fn spawn_ws_consumer(&mut self) -> anyhow::Result<()> {
1147 let mut ws_guard = TaskRetentionGuard::new(
1151 self.ws_client.clone(),
1152 Arc::clone(&self.ws_handler_retained),
1153 );
1154 ws_guard
1155 .client_mut()
1156 .connect_with_cancellation(self.cancellation_token.clone())
1157 .await
1158 .context("failed to connect to Lighter WebSocket")?;
1159
1160 let post_connect = async {
1164 ws_guard
1165 .client_mut()
1166 .wait_until_active()
1167 .await
1168 .context("Lighter WebSocket did not reach active state")?;
1169
1170 if let Some(credential) = &self.credential {
1171 let auth_token = build_auth_token_for(credential)
1172 .context("failed to mint Lighter auth token")?;
1173 let account_index = credential.account_index();
1174
1175 ws_guard
1176 .client_mut()
1177 .set_execution_context(self.core.account_id, account_index)
1178 .await
1179 .map_err(|e| anyhow::anyhow!("failed to set Lighter execution context: {e}"))?;
1180
1181 let channels = [
1186 LighterWsChannel::AccountAllOrders(account_index),
1187 LighterWsChannel::AccountAllTrades(account_index),
1188 LighterWsChannel::AccountAllPositions(account_index),
1189 LighterWsChannel::AccountAllAssets(account_index),
1190 LighterWsChannel::UserStats(account_index),
1191 ];
1192
1193 for channel in channels {
1194 ws_guard
1195 .client_mut()
1196 .subscribe_account(channel.clone(), auth_token.clone())
1197 .await
1198 .map_err(|e| {
1199 anyhow::anyhow!(
1200 "failed to subscribe to Lighter account channel {channel:?}: {e}",
1201 )
1202 })?;
1203 }
1204
1205 log::debug!("Subscribed to Lighter account streams: account_index={account_index}",);
1206 } else {
1207 log::warn!(
1208 "Lighter execution client has no credentials: account streams not subscribed; \
1209 typed execution reports will not flow"
1210 );
1211 }
1212
1213 Ok::<(), anyhow::Error>(())
1214 };
1215
1216 if let Err(e) = post_connect.await {
1217 log::warn!("Lighter post-connect setup failed, tearing down WS: {e}");
1218 let ws_client = ws_guard.disarm();
1219 let mut rollback_errors = Vec::new();
1220
1221 if let Err(e) = ws_client
1222 .disconnect_with_task_retention(Arc::clone(&self.ws_handler_retained))
1223 .await
1224 {
1225 rollback_errors.push(e.to_string());
1226 }
1227
1228 if let Err(e) = self.ws_handler_retained.finish().await {
1229 rollback_errors.push(e.to_string());
1230 }
1231
1232 if rollback_errors.is_empty() {
1233 return Err(e);
1234 }
1235 return Err(e.context(format!(
1236 "Lighter post-connect rollback failed: {}",
1237 rollback_errors.join("; ")
1238 )));
1239 }
1240
1241 let mut ws_client = ws_guard.disarm();
1242 self.ws_client.set_task_slot(ws_client.take_task_slot());
1243
1244 let cancellation_token = self.cancellation_token.clone();
1245 let emitter = self.emitter.clone();
1246 let dispatch = self.dispatch.clone();
1247 let credential_for_loop = self.credential.clone();
1248 let http_client_for_loop = self.http_client.clone();
1249 let registry_for_loop = Arc::clone(&self.registry);
1250 let auth_refresh_notify = Arc::clone(&self.auth_refresh_notify);
1251 let nonce_submission_gate = Arc::clone(&self.nonce_submission_gate);
1252 let nonce_ready_connection_epoch = Arc::clone(&self.nonce_ready_connection_epoch);
1253 let scheduled_cancel_notify = Arc::clone(&self.scheduled_cancel_notify);
1254 let pending_tasks_for_loop = self
1255 .pending_tasks
1256 .spawner()
1257 .context("Lighter task admission is closed")?;
1258 let account_id_for_loop = self.core.account_id;
1259 let clock_for_loop = self.clock;
1260
1261 let orders_cached = {
1262 let cache = self.core.cache();
1263 cache
1264 .orders_open_refs(
1265 Some(&self.core.venue),
1266 None,
1267 None,
1268 Some(&self.core.account_id),
1269 None,
1270 )
1271 .into_iter()
1272 .map(|order| order.clone())
1273 .collect()
1274 };
1275
1276 dispatch.seed_order_restorations(orders_cached);
1277
1278 let nonce_refresh_retry = NonceRefreshRetry {
1279 http_client: self.http_client.clone(),
1280 dispatch: self.dispatch.clone(),
1281 credential: self.credential.clone(),
1282 submission_gate: Arc::clone(&self.nonce_submission_gate),
1283 ready_connection_epoch: Arc::clone(&self.nonce_ready_connection_epoch),
1284 ws_client: ws_client.clone(),
1285 cancellation_token: self.cancellation_token.clone(),
1286 pending_tasks: pending_tasks_for_loop.clone(),
1287 };
1288
1289 let child_context = credential_for_loop
1290 .as_ref()
1291 .map(|credential| AckedOrderProbeContext {
1292 http_client: http_client_for_loop.clone(),
1293 registry: Arc::clone(®istry_for_loop),
1294 credential: credential.clone(),
1295 dispatch: dispatch.clone(),
1296 account_id: account_id_for_loop,
1297 clock: clock_for_loop,
1298 emitter: emitter.clone(),
1299 connection_epoch: ws_client.connection_epoch_atomic(),
1300 cancellation_token: cancellation_token.clone(),
1301 pending_tasks: pending_tasks_for_loop.clone(),
1302 });
1303
1304 if let Some(context) = child_context.clone() {
1305 self.terminal_producers_stopped = CancellationToken::new();
1306
1307 self.terminal_recovery_handle.spawn(run_terminal_recovery(
1308 context,
1309 self.terminal_producers_stopped.clone(),
1310 ))?;
1311 }
1312
1313 self.ws_stream_handle.spawn(async move {
1314 log::debug!("Lighter execution WebSocket consumption loop started");
1315
1316 loop {
1317 tokio::select! {
1318 biased;
1319 () = cancellation_token.cancelled() => {
1320 log::debug!("Lighter execution consumption loop cancelled");
1321 break;
1322 }
1323 msg_opt = ws_client.next_event() => {
1324 match msg_opt {
1325 Some(NautilusWsMessage::ExecutionReports(reports)) => {
1326 let mut order_count = 0_usize;
1327 let mut fill_count = 0_usize;
1328 let trader_id = emitter.trader_id();
1329 let account_index = credential_for_loop
1330 .as_ref()
1331 .map(|c| c.account_index());
1332
1333 for report in reports {
1334 match report {
1335 LighterExecutionReport::Order(order) => {
1336 order_count += 1;
1337 dispatch_lighter_order(
1338 &order,
1339 &dispatch,
1340 &emitter,
1341 ®istry_for_loop,
1342 account_id_for_loop,
1343 trader_id,
1344 clock_for_loop.get_time_ns(),
1345 );
1346 }
1347 LighterExecutionReport::Fill(trade) => {
1348 fill_count += 1;
1349 dispatch_lighter_trade(
1350 &trade,
1351 &dispatch,
1352 &emitter,
1353 ®istry_for_loop,
1354 account_id_for_loop,
1355 trader_id,
1356 account_index,
1357 clock_for_loop.get_time_ns(),
1358 );
1359 }
1360 }
1361 }
1362
1363 if let Some(context) = &child_context {
1364 spawn_child_recoveries(context);
1365 }
1366 log::debug!(
1367 "Lighter execution batch: orders={order_count} fills={fill_count}",
1368 );
1369 }
1370 Some(NautilusWsMessage::PositionSnapshot {
1371 reports,
1372 skipped_market_ids,
1373 }) => {
1374 for r in &reports {
1377 if let Some(idx) =
1378 registry_for_loop.market_index(&r.instrument_id)
1379 {
1380 dispatch.note_active_market(idx);
1381 }
1382 }
1383 let position_count = reports.len();
1384 let retained_positions: Vec<InstrumentId> = skipped_market_ids
1385 .iter()
1386 .filter_map(|market_id| {
1387 registry_for_loop.instrument_id(*market_id)
1388 })
1389 .collect();
1390 let removed = dispatch.replace_position_snapshot(
1391 &reports,
1392 &retained_positions,
1393 &skipped_market_ids,
1394 );
1395 log::debug!(
1396 "Lighter position snapshot: positions={position_count}, skipped_markets={}, removed={}",
1397 skipped_market_ids.len(),
1398 removed.len(),
1399 );
1400 emit_lighter_position_reports(
1401 reports,
1402 removed,
1403 &emitter,
1404 account_id_for_loop,
1405 clock_for_loop.get_time_ns(),
1406 );
1407 }
1408 Some(NautilusWsMessage::PositionUpdate {
1409 reports,
1410 closed_market_ids,
1411 skipped_market_ids,
1412 }) => {
1413 let mut covered_market_ids = closed_market_ids.clone();
1414
1415 for report in &reports {
1416 if let Some(market_id) =
1417 registry_for_loop.market_index(&report.instrument_id)
1418 {
1419 dispatch.note_active_market(market_id);
1420 covered_market_ids.push(market_id);
1421 }
1422 }
1423 let position_count = reports.len();
1424 let closed_positions: Vec<InstrumentId> = closed_market_ids
1425 .iter()
1426 .filter_map(|market_id| {
1427 registry_for_loop.instrument_id(*market_id)
1428 })
1429 .collect();
1430 let removed = dispatch.apply_position_update(
1431 &reports,
1432 &closed_positions,
1433 &covered_market_ids,
1434 &skipped_market_ids,
1435 );
1436 log::debug!(
1437 "Lighter position update: positions={position_count}, closed_markets={}, skipped_markets={}, removed={}",
1438 closed_market_ids.len(),
1439 skipped_market_ids.len(),
1440 removed.len(),
1441 );
1442 emit_lighter_position_reports(
1443 reports,
1444 removed,
1445 &emitter,
1446 account_id_for_loop,
1447 clock_for_loop.get_time_ns(),
1448 );
1449 }
1450 Some(NautilusWsMessage::AccountState(state)) => {
1451 log::debug!(
1452 "Lighter AccountState: balances={} margins={}",
1453 state.balances.len(),
1454 state.margins.len(),
1455 );
1456 dispatch.cache_account_state((*state).clone());
1462 emitter.send_account_state(*state);
1463 }
1464 Some(NautilusWsMessage::Reconnected { connection_epoch }) => {
1465 dispatch.terminals_recover.notify_one();
1466 dispatch.invalidate_position_snapshot();
1467 dispatch.stop_cancel_probes_before(connection_epoch);
1468 nonce_ready_connection_epoch.store(
1469 NONCE_CONNECTION_EPOCH_UNAVAILABLE,
1470 Ordering::Release,
1471 );
1472 log::debug!("Lighter WebSocket reconnected (execution stream)");
1480 auth_refresh_notify.notify_one();
1481
1482 let refresh_guard = nonce_submission_gate.write().await;
1496 let disconnected_epoch = connection_epoch.saturating_sub(1);
1497 let stale = dispatch.drain_pending_sendtx(disconnected_epoch);
1498 warn_pending_sendtx_unknown(&stale, "reconnect");
1499
1500 if let Some(credential) = &credential_for_loop {
1501 match http_client_for_loop
1502 .get_next_nonce(
1503 credential.account_index(),
1504 credential.api_key_index(),
1505 )
1506 .await
1507 {
1508 Ok(response) => {
1509 dispatch.nonce_manager.refresh(
1510 credential.account_index(),
1511 credential.api_key_index(),
1512 response.nonce,
1513 );
1514 nonce_ready_connection_epoch
1515 .store(connection_epoch, Ordering::Release);
1516 drop(refresh_guard);
1517 scheduled_cancel_notify.notify_one();
1518 log::debug!(
1519 "Refreshed Lighter nonce after reconnect: \
1520 account_index={}, next_nonce={}",
1521 credential.account_index(),
1522 response.nonce,
1523 );
1524 }
1525 Err(e) => {
1526 log::warn!(
1527 "Failed to refresh Lighter nonce after reconnect: {e}",
1528 );
1529 nonce_refresh_retry
1530 .clone()
1531 .spawn(connection_epoch);
1532 }
1533 }
1534 }
1535 }
1536 Some(message @ (NautilusWsMessage::SendTxAck { .. } | NautilusWsMessage::SendTxBatchResult { .. })) => {
1537 let account_index = credential_for_loop.as_ref().map(|c| c.account_index());
1538 let acked = match message {
1539 NautilusWsMessage::SendTxAck { connection_epoch, tx_hash, code } => {
1540 handle_send_tx_ack_for_connection(&dispatch, account_index, connection_epoch, code, tx_hash.as_deref()).into_iter().collect()
1541 }
1542 NautilusWsMessage::SendTxBatchResult { connection_epoch, id, code, message, tx_hashes } => {
1543 let (acked, resync) = handle_send_tx_batch_result(
1544 &dispatch, &emitter, account_index, connection_epoch,
1545 clock_for_loop.get_time_ns(), &id, code, &message, &tx_hashes,
1546 );
1547
1548 if resync && ws_client.connection_epoch() == connection_epoch {
1549 nonce_ready_connection_epoch.store(NONCE_CONNECTION_EPOCH_UNAVAILABLE, Ordering::Release);
1550
1551 nonce_refresh_retry.clone().spawn(connection_epoch);
1552 }
1553 acked
1554 }
1555 _ => unreachable!("matched transaction response"),
1556 };
1557
1558 for pending in acked {
1559 let Some(credential) = credential_for_loop.clone() else { continue; };
1560
1561 if matches!(pending.kind, PendingSendTxKind::Modify { .. }) { continue; }
1562 spawn_acked_order_probe(
1563 &pending,
1564 AckedOrderProbeContext {
1565 http_client: http_client_for_loop.clone(),
1566 registry: Arc::clone(®istry_for_loop),
1567 credential,
1568 dispatch: dispatch.clone(),
1569 account_id: account_id_for_loop,
1570 clock: clock_for_loop,
1571 emitter: emitter.clone(),
1572 connection_epoch: ws_client.connection_epoch_atomic(),
1573 cancellation_token: cancellation_token.clone(),
1574 pending_tasks: pending_tasks_for_loop.clone(),
1575 },
1576 );
1577 }
1578 }
1579 Some(NautilusWsMessage::SendTxRejected {
1580 connection_epoch,
1581 source,
1582 code,
1583 message,
1584 tx_hash,
1585 }) => {
1586 let account_index = credential_for_loop
1587 .as_ref()
1588 .map(|c| c.account_index());
1589 let needs_nonce_resync = handle_send_tx_rejection_for_connection(
1590 &dispatch,
1591 &emitter,
1592 account_index,
1593 connection_epoch,
1594 clock_for_loop.get_time_ns(),
1595 source,
1596 code,
1597 &message,
1598 tx_hash.as_deref(),
1599 );
1600
1601 if needs_nonce_resync
1605 && let Some(credential) = &credential_for_loop
1606 {
1607 if ws_client.connection_epoch() != connection_epoch {
1608 log::warn!(
1609 "Skipping stale Lighter nonce rejection from connection \
1610 epoch {connection_epoch}",
1611 );
1612 continue;
1613 }
1614 nonce_ready_connection_epoch.store(
1615 NONCE_CONNECTION_EPOCH_UNAVAILABLE,
1616 Ordering::Release,
1617 );
1618 let _refresh_guard = nonce_submission_gate.write().await;
1619
1620 match http_client_for_loop
1621 .get_next_nonce(
1622 credential.account_index(),
1623 credential.api_key_index(),
1624 )
1625 .await
1626 {
1627 Ok(response) => {
1628 dispatch.nonce_manager.refresh(
1629 credential.account_index(),
1630 credential.api_key_index(),
1631 response.nonce,
1632 );
1633 nonce_ready_connection_epoch
1634 .store(connection_epoch, Ordering::Release);
1635 log::debug!(
1636 "Hard-refreshed Lighter nonce after invalid-nonce \
1637 rejection: account_index={}, next_nonce={}",
1638 credential.account_index(),
1639 response.nonce,
1640 );
1641 }
1642 Err(e) => {
1643 log::warn!(
1644 "Failed to refresh Lighter nonce after \
1645 invalid-nonce rejection: {e}",
1646 );
1647 nonce_refresh_retry
1648 .clone()
1649 .spawn(connection_epoch);
1650 }
1651 }
1652 }
1653 }
1654 Some(NautilusWsMessage::Raw(value)) => {
1655 log::debug!("Unhandled Lighter raw frame on execution stream: {value}");
1656 }
1657 Some(NautilusWsMessage::AccountStreamFirstFrame(stream)) => {
1658 match stream {
1664 AccountStream::Orders => {
1665 dispatch.account_streams_ready.mark_orders();
1666 }
1667 AccountStream::Trades => {
1668 dispatch.account_streams_ready.mark_trades();
1669 }
1670 AccountStream::Positions => {
1671 dispatch.account_streams_ready.mark_positions();
1672 }
1673 AccountStream::Assets => {
1674 dispatch.account_streams_ready.mark_assets();
1675 }
1676 AccountStream::UserStats => {
1677 dispatch.account_streams_ready.mark_user_stats();
1678 }
1679 }
1680 }
1681 Some(
1685 NautilusWsMessage::Trades(_)
1686 | NautilusWsMessage::Quote(_)
1687 | NautilusWsMessage::Deltas(_)
1688 | NautilusWsMessage::Depth(_)
1689 | NautilusWsMessage::Bar(_)
1690 | NautilusWsMessage::MarkPrice(_)
1691 | NautilusWsMessage::IndexPrice(_)
1692 | NautilusWsMessage::FundingRate(_),
1693 ) => {}
1694 None => {
1695 log::debug!("Lighter execution next_event returned None");
1696 tokio::select! {
1697 () = cancellation_token.cancelled() => {
1698 log::debug!(
1699 "Lighter execution consumption loop cancelled"
1700 );
1701 break;
1702 }
1703 () = tokio::time::sleep(Duration::from_secs(1)) => {}
1704 }
1705 }
1706 }
1707 }
1708 }
1709 }
1710
1711 log::debug!("Lighter execution WebSocket consumption loop finished");
1712 })?;
1713
1714 if let Some(credential) = &self.credential {
1715 self.spawn_auth_token_refresh(credential.clone())?;
1716 }
1717
1718 Ok(())
1719 }
1720
1721 fn spawn_auth_token_refresh(&mut self, credential: Credential) -> anyhow::Result<()> {
1722 let ws_client = self.ws_client.clone();
1723 let cancellation_token = self.cancellation_token.clone();
1724 let account_index = credential.account_index();
1725 let channels = auth_token_rotation_channels(account_index);
1726 let refresh_notify = Arc::clone(&self.auth_refresh_notify);
1727
1728 self.auth_refresh_handle.spawn(async move {
1729 log::debug!(
1730 "Lighter auth-token refresh task started: interval={}s, account_index={account_index}",
1731 AUTH_TOKEN_REFRESH_INTERVAL.as_secs(),
1732 );
1733
1734 let mut next_refresh_delay = AUTH_TOKEN_REFRESH_INTERVAL;
1735
1736 loop {
1737 tokio::select! {
1738 biased;
1739 () = cancellation_token.cancelled() => {
1740 log::debug!("Lighter auth-token refresh task cancelled");
1741 break;
1742 }
1743 () = refresh_notify.notified() => {
1744 log::debug!(
1745 "Refreshing Lighter auth token immediately after reconnect: account_index={account_index}",
1746 );
1747 }
1748 () = tokio::time::sleep(next_refresh_delay) => {}
1749 }
1750
1751 let outcome = refresh_auth_token_until_rotated(
1752 &credential,
1753 &channels,
1754 &cancellation_token,
1755 AUTH_TOKEN_REFRESH_BACKOFF,
1756 build_auth_token_for,
1757 |channel, token| {
1758 let ws_client = ws_client.clone();
1759 async move { ws_client.subscribe_account(channel, token).await }
1760 },
1761 )
1762 .await;
1763
1764 if let Some(delay) = auth_token_refresh_next_delay(outcome) {
1765 if outcome == AuthTokenRefreshOutcome::Exhausted {
1766 log::error!(
1767 "Lighter auth-token rotation exhausted retry window; retrying again in {}s: account_index={account_index}",
1768 delay.as_secs(),
1769 );
1770 }
1771 next_refresh_delay = delay;
1772 } else {
1773 log::debug!("Lighter auth-token refresh task cancelled");
1774 break;
1775 }
1776 }
1777 })?;
1778 Ok(())
1779 }
1780
1781 fn resolve_slippage_bps(&self, params: Option<&Params>) -> u32 {
1783 params
1784 .and_then(|p| p.get_u64("market_order_slippage_bps"))
1785 .map_or(self.config.market_order_slippage_bps, |v| v as u32)
1786 }
1787
1788 fn build_tx_context(&self, credential: &Credential) -> anyhow::Result<ReservedTxContext> {
1789 let connection_epoch = self.ws_client.connection_epoch();
1790 anyhow::ensure!(
1791 self.nonce_ready_connection_epoch.load(Ordering::Acquire) == connection_epoch,
1792 TxPreparationError::NonceRefreshPending(connection_epoch),
1793 );
1794 let nonce_guard = Arc::clone(&self.nonce_submission_gate)
1795 .try_read_owned()
1796 .context("Lighter nonce refresh is in progress")?;
1797 anyhow::ensure!(
1798 self.ws_client.connection_epoch() == connection_epoch
1799 && self.nonce_ready_connection_epoch.load(Ordering::Acquire) == connection_epoch,
1800 TxPreparationError::StateChanged,
1801 );
1802
1803 let send_reservation = match self.tx_send_sequencer.reserve(
1804 credential.account_index(),
1805 credential.api_key_index(),
1806 || {
1807 self.dispatch
1808 .nonce_manager
1809 .next_nonce(credential.account_index(), credential.api_key_index())
1810 },
1811 ) {
1812 Ok(reservation) => reservation,
1813 Err(e @ NonceError::SkipWindowExhausted { .. }) => {
1814 self.spawn_nonce_window_recovery(credential);
1815 anyhow::bail!("failed to allocate Lighter nonce: {e}");
1816 }
1817 Err(e) => anyhow::bail!("failed to allocate Lighter nonce: {e}"),
1818 };
1819
1820 let nonce = send_reservation.nonce;
1821
1822 let now_ns = self.clock.get_time_ns().as_u64() as i64;
1823 let expired_at = (now_ns / 1_000_000).saturating_add(DEFAULT_TX_EXPIRY_MS);
1824 let send_reservation = send_reservation.with_nonce_ownership(nonce_guard, connection_epoch);
1825
1826 let context = TxContext {
1827 account_index: credential.account_index(),
1828 api_key_index: credential.api_key_index(),
1829 nonce,
1830 expired_at,
1831 };
1832
1833 Ok(ReservedTxContext {
1834 context,
1835 send_reservation,
1836 })
1837 }
1838
1839 fn spawn_nonce_window_recovery(&self, credential: &Credential) {
1840 if self.nonce_recovery_inflight.swap(true, Ordering::AcqRel) {
1841 return;
1842 }
1843
1844 let inflight = Arc::clone(&self.nonce_recovery_inflight);
1845 let http_client = self.http_client.clone();
1846 let dispatch = self.dispatch.clone();
1847 let nonce_submission_gate = Arc::clone(&self.nonce_submission_gate);
1848 let account_index = credential.account_index();
1849 let api_key_index = credential.api_key_index();
1850
1851 self.spawn_task("nonce_window_recovery", async move {
1852 let _refresh_guard = nonce_submission_gate.write().await;
1853 let result = http_client
1854 .get_next_nonce(account_index, api_key_index)
1855 .await;
1856 inflight.store(false, Ordering::Release);
1857
1858 match result {
1859 Ok(response) => {
1860 let _ = dispatch.nonce_manager.sync_from_venue(
1863 account_index,
1864 api_key_index,
1865 response.nonce,
1866 );
1867 log::debug!(
1868 "Resynced Lighter nonce baseline after skip-window exhaustion: \
1869 account_index={account_index}, api_key_index={api_key_index}, \
1870 next_nonce={}",
1871 response.nonce,
1872 );
1873 }
1874 Err(e) => {
1875 log::error!(
1876 "Failed to resync Lighter nonce after skip-window exhaustion: {e}",
1877 );
1878 }
1879 }
1880 Ok(())
1881 });
1882 }
1883
1884 fn dispatch_create_order_plan(
1885 &self,
1886 plan: &CreateOrderPlan,
1887 credential: &Credential,
1888 ) -> anyhow::Result<()> {
1889 let context = self.fanout_dispatch_context(credential)?;
1890 let prepared = context.sign_create_order(plan)?;
1891 self.spawn_task("submit_order", async move {
1892 context.send_create_order(prepared).await;
1893 Ok(())
1894 });
1895
1896 Ok(())
1897 }
1898
1899 fn prepare_create_order_plan(
1900 &self,
1901 order: &OrderAny,
1902 slippage_bps: u32,
1903 params: Option<&Params>,
1904 ) -> anyhow::Result<CreateOrderPlan> {
1905 let attributes = self.resolve_create_attributes(params)?;
1906 let instrument_id = order.instrument_id();
1907 let market_index = self.registry.market_index(&instrument_id).ok_or_else(|| {
1908 anyhow::anyhow!("no Lighter market_index registered for instrument {instrument_id}")
1909 })?;
1910
1911 let instrument = self.core.cache().try_instrument(&instrument_id)?.clone();
1912
1913 let order_kind = nautilus_to_lighter_order_type(order.order_type())?;
1914
1915 let tif = nautilus_to_lighter_tif(
1916 order.order_type(),
1917 order.time_in_force(),
1918 order.is_post_only(),
1919 )?;
1920 let now_ms = (self.clock.get_time_ns().as_u64() / 1_000_000) as i64;
1921 let order_expiry = order_expiry_for(
1922 order.order_type(),
1923 &order.time_in_force(),
1924 order.expire_time(),
1925 now_ms,
1926 self.config.use_gtd,
1927 )?;
1928
1929 let base_amount = quantity_to_ticks(&order.quantity(), instrument.size_precision())?;
1930
1931 anyhow::ensure!(
1933 base_amount > 0,
1934 "quantity `{}` rounds to 0 ticks at size_precision {}",
1935 order.quantity(),
1936 instrument.size_precision(),
1937 );
1938 let price_precision = instrument.price_precision();
1939 let is_buy = matches!(order.order_side(), OrderSide::Buy);
1940
1941 let price_ticks = match order.order_type() {
1944 OrderType::Market => {
1945 let quote = self
1946 .core
1947 .cache()
1948 .quote(&instrument_id)
1949 .copied()
1950 .ok_or_else(|| {
1951 anyhow::anyhow!(
1952 "no cached quote for {instrument_id}: subscribe to quotes before submitting MARKET orders",
1953 )
1954 })?;
1955 let base = if is_buy {
1956 quote.ask_price.as_decimal()
1957 } else {
1958 quote.bid_price.as_decimal()
1959 };
1960 derive_market_order_price_ticks(base, is_buy, price_precision, slippage_bps)?
1961 }
1962 OrderType::StopMarket | OrderType::MarketIfTouched => {
1963 let trigger = order.trigger_price().ok_or_else(|| {
1964 anyhow::anyhow!("{:?} orders require a trigger_price", order.order_type(),)
1965 })?;
1966 derive_market_order_price_ticks(
1967 trigger.as_decimal(),
1968 is_buy,
1969 price_precision,
1970 slippage_bps,
1971 )?
1972 }
1973 _ => order
1974 .price()
1975 .map(|p| price_to_ticks(&p, price_precision))
1976 .transpose()?
1977 .unwrap_or(0),
1978 };
1979
1980 let trigger_price_ticks = order
1981 .trigger_price()
1982 .map(|p| price_to_ticks(&p, price_precision))
1983 .transpose()?
1984 .unwrap_or(0);
1985
1986 if matches!(
1989 order.order_type(),
1990 OrderType::StopMarket
1991 | OrderType::StopLimit
1992 | OrderType::MarketIfTouched
1993 | OrderType::LimitIfTouched
1994 ) {
1995 anyhow::ensure!(
1996 trigger_price_ticks > 0,
1997 "trigger_price `{:?}` rounds to 0 ticks at precision {price_precision}",
1998 order.trigger_price(),
1999 );
2000 }
2001 validate_order_amount(&instrument, order.quantity(), price_ticks, price_precision)?;
2002
2003 Ok(CreateOrderPlan {
2004 order: order.clone(),
2005 market_index,
2006 base_amount,
2007 price: price_ticks,
2008 order_type: order_kind as u8,
2009 time_in_force: tif as u8,
2010 trigger_price: trigger_price_ticks,
2011 order_expiry,
2012 attributes,
2013 })
2014 }
2015
2016 fn resolve_create_attributes(&self, params: Option<&Params>) -> anyhow::Result<L2TxAttributes> {
2017 let mut attributes = integrator_attributes(self.integrator_account_index());
2018
2019 let behavior = match params.and_then(|p| p.get("self_trade_behavior_mode")) {
2020 Some(value) => value
2021 .as_str()
2022 .context("Lighter self_trade_behavior_mode must be a string")?
2023 .parse::<LighterSelfTradeBehaviorMode>()
2024 .context("invalid Lighter self_trade_behavior_mode")?,
2025 None => LighterSelfTradeBehaviorMode::default(),
2026 };
2027
2028 let equality = match params.and_then(|p| p.get("self_trade_equality_mode")) {
2029 Some(value) => value
2030 .as_str()
2031 .context("Lighter self_trade_equality_mode must be a string")?
2032 .parse::<LighterSelfTradeEqualityMode>()
2033 .context("invalid Lighter self_trade_equality_mode")?,
2034 None => LighterSelfTradeEqualityMode::default(),
2035 };
2036
2037 attributes.self_trade_behavior_mode = Some(behavior);
2038 attributes.self_trade_equality_mode = Some(equality);
2039 attributes.validate()?;
2040 Ok(attributes)
2041 }
2042
2043 fn prepare_create_group_plan(
2044 &self,
2045 orders: &[OrderAny],
2046 slippage_bps: u32,
2047 params: Option<&Params>,
2048 ) -> anyhow::Result<CreateGroupPlan> {
2049 let (grouping_type, indices) = grouped_order_indices(orders)?;
2050 let mut plans = Vec::with_capacity(indices.len());
2051 for index in indices {
2052 let order = &orders[index];
2053 let instrument = self
2054 .core
2055 .cache()
2056 .instrument(&order.instrument_id())
2057 .cloned();
2058
2059 if let Some(reason) = local_submit_denial_reason(order, instrument.as_ref()) {
2060 return Err(reason);
2061 }
2062
2063 anyhow::ensure!(
2064 !is_lighter_spot_order(order, instrument.as_ref()),
2065 "Lighter grouped orders require perpetual markets"
2066 );
2067 plans.push(self.prepare_create_order_plan(order, slippage_bps, params)?);
2068 }
2069
2070 prepare_group_bodies(grouping_type, &mut plans, self.config.use_gtd)?;
2071 Ok(CreateGroupPlan {
2072 grouping_type,
2073 orders: plans,
2074 })
2075 }
2076
2077 fn fanout_dispatch_context(
2078 &self,
2079 credential: &Credential,
2080 ) -> anyhow::Result<FanoutDispatchContext> {
2081 let pending_tasks = self
2082 .pending_tasks
2083 .spawner()
2084 .context("Lighter task admission is closed")?;
2085 Ok(FanoutDispatchContext {
2086 clock: self.clock,
2087 chain_id: self.config.chain_id(),
2088 integrator_account_index: self.integrator_account_index(),
2089 emitter: self.emitter.clone(),
2090 credential: credential.clone(),
2091 http_client: self.http_client.clone(),
2092 registry: Arc::clone(&self.registry),
2093 ws_client: self.ws_client.clone(),
2094 tx_rate_limiter: Arc::clone(&self.tx_rate_limiter),
2095 tx_send_sequencer: self.tx_send_sequencer.clone(),
2096 nonce_submission_gate: Arc::clone(&self.nonce_submission_gate),
2097 nonce_ready_connection_epoch: Arc::clone(&self.nonce_ready_connection_epoch),
2098 dispatch: self.dispatch.clone(),
2099 nonce_recovery_inflight: Arc::clone(&self.nonce_recovery_inflight),
2100 pending_tasks,
2101 })
2102 }
2103
2104 fn dispatch_signed_cancel_order(&self, cmd: &CancelOrder, credential: &Credential) {
2105 let context = match self.fanout_dispatch_context(credential) {
2106 Ok(context) => context,
2107 Err(e) => {
2108 log::warn!("Skipping Lighter cancel_order after shutdown began: {e}");
2109 return;
2110 }
2111 };
2112 self.dispatch
2113 .set_pending_order_action(cmd.client_order_id, PendingOrderAction::Cancel);
2114 let emit_cancel_rejected = self.can_emit_order_cancel_rejected(&cmd.client_order_id);
2115 if !emit_cancel_rejected {
2116 self.dispatch
2117 .clear_pending_order_action_if(&cmd.client_order_id, PendingOrderAction::Cancel);
2118 }
2119
2120 let prepared = match self
2121 .prepare_cancel_order_plan(cmd)
2122 .and_then(|plan| context.sign_cancel_order(&plan))
2123 {
2124 Ok(prepared) => prepared,
2125 Err(e) => {
2126 self.dispatch.clear_pending_order_action_if(
2127 &cmd.client_order_id,
2128 PendingOrderAction::Cancel,
2129 );
2130 let reason = format!("Lighter cancel_order failed: {e}");
2131 if emit_cancel_rejected {
2132 log::warn!("{reason} for {}", cmd.client_order_id);
2133 self.emitter.emit_order_cancel_rejected_event(
2134 cmd.strategy_id,
2135 cmd.instrument_id,
2136 cmd.client_order_id,
2137 cmd.venue_order_id,
2138 &reason,
2139 self.clock.get_time_ns(),
2140 );
2141 } else {
2142 log::warn!(
2143 "{reason} for {}; suppressing OrderCancelRejected because order is not PendingCancel",
2144 cmd.client_order_id,
2145 );
2146 }
2147 return;
2148 }
2149 };
2150
2151 self.spawn_task("cancel_order", async move {
2152 context
2153 .send_cancel_order(prepared, emit_cancel_rejected)
2154 .await;
2155 Ok(())
2156 });
2157 }
2158
2159 fn dispatch_cancel_orders(&self, cancels: &[CancelOrder]) -> anyhow::Result<()> {
2160 let credential = self.credential.as_ref().ok_or_else(|| {
2161 anyhow::anyhow!("Lighter execution client cannot cancel without credentials")
2162 })?;
2163
2164 if cancels.is_empty() {
2165 log::debug!("batch_cancel_orders called with empty cancel list");
2166 return Ok(());
2167 }
2168
2169 let context = self.fanout_dispatch_context(credential)?;
2170
2171 let emit_cancel_rejected: Vec<bool> = cancels
2172 .iter()
2173 .map(|cancel| self.can_emit_order_cancel_rejected(&cancel.client_order_id))
2174 .collect();
2175
2176 let mut plans = Vec::with_capacity(cancels.len());
2177
2178 for (cancel, emit_cancel_rejected) in cancels.iter().zip(emit_cancel_rejected) {
2179 let action = PendingOrderAction::cancel(emit_cancel_rejected);
2180 self.dispatch
2181 .set_pending_order_action(cancel.client_order_id, action);
2182
2183 match self.prepare_cancel_order_plan(cancel) {
2184 Ok(plan) => {
2185 let group = self
2186 .prepare_oco_cancel_plans(cancel)
2187 .unwrap_or_else(|| vec![plan]);
2188 plans.extend(group.into_iter().map(|plan| {
2189 let emit = self.can_emit_order_cancel_rejected(&plan.client_order_id);
2190 (plan, emit)
2191 }));
2192 }
2193 Err(e) => {
2194 self.dispatch
2195 .clear_pending_order_action_if(&cancel.client_order_id, action);
2196 let reason = format!("Lighter cancel_order failed: {e}");
2197
2198 if emit_cancel_rejected {
2199 self.emitter.emit_order_cancel_rejected_event(
2200 cancel.strategy_id,
2201 cancel.instrument_id,
2202 cancel.client_order_id,
2203 cancel.venue_order_id,
2204 &reason,
2205 self.clock.get_time_ns(),
2206 );
2207 } else {
2208 log::warn!(
2209 "{reason} for {}; suppressing OrderCancelRejected because order is not PendingCancel",
2210 cancel.client_order_id,
2211 );
2212 }
2213 }
2214 }
2215 }
2216
2217 let mut seen = AHashSet::new();
2218 plans.retain(|(plan, _)| seen.insert(plan.client_order_id));
2219 for (plan, emit) in &plans {
2220 self.dispatch
2221 .set_pending_order_action(plan.client_order_id, PendingOrderAction::cancel(*emit));
2222 }
2223
2224 if plans.is_empty() {
2225 return Ok(());
2226 }
2227
2228 self.spawn_task("batch_cancel_orders", async move {
2229 context
2230 .dispatch_batch_plans(
2231 plans
2232 .into_iter()
2233 .map(|(plan, emit)| BatchOrderPlan::Cancel(plan, emit))
2234 .collect(),
2235 )
2236 .await;
2237 Ok(())
2238 });
2239
2240 Ok(())
2241 }
2242
2243 fn can_emit_order_cancel_rejected(&self, client_order_id: &ClientOrderId) -> bool {
2244 self.core
2245 .cache()
2246 .order(client_order_id)
2247 .is_none_or(|order| order.is_pending_cancel())
2248 }
2249
2250 fn prepare_cancel_order_plan(&self, cmd: &CancelOrder) -> anyhow::Result<CancelOrderPlan> {
2251 let market_index = self
2252 .registry
2253 .market_index(&cmd.instrument_id)
2254 .ok_or_else(|| {
2255 anyhow::anyhow!(
2256 "no Lighter market_index registered for instrument {}",
2257 cmd.instrument_id,
2258 )
2259 })?;
2260
2261 self.core.cache().try_order(&cmd.client_order_id)?;
2262
2263 let voi = cmd
2266 .venue_order_id
2267 .or_else(|| self.dispatch.lookup_venue_order_id(&cmd.client_order_id))
2268 .ok_or_else(|| {
2269 anyhow::anyhow!(
2270 "cannot cancel Lighter order {}: venue order_id not yet known \
2271 (await OrderAccepted before issuing cancel)",
2272 cmd.client_order_id,
2273 )
2274 })?;
2275
2276 let venue_index: i64 = voi
2277 .as_str()
2278 .parse()
2279 .with_context(|| format!("Lighter venue_order_id `{voi}` is not an integer index"))?;
2280
2281 Ok(CancelOrderPlan {
2282 client_order_id: cmd.client_order_id,
2283 strategy_id: cmd.strategy_id,
2284 instrument_id: cmd.instrument_id,
2285 venue_order_id: Some(voi),
2286 market_index,
2287 venue_index,
2288 })
2289 }
2290
2291 fn dispatch_signed_modify_order(&self, cmd: &ModifyOrder, credential: &Credential) {
2292 let probe_context = match self.fanout_dispatch_context(credential) {
2293 Ok(context) => context.probe_context(),
2294 Err(e) => {
2295 log::warn!("Lighter modify probe unavailable: {e}");
2296 return;
2297 }
2298 };
2299
2300 if !self.admit_modify_order(cmd) {
2301 return;
2302 }
2303
2304 let prepared = match self.prepare_signed_modify_order(cmd, credential) {
2305 Ok(prepared) => prepared,
2306 Err(e) => {
2307 let mut events = self.dispatch.order_events.lock();
2308 self.dispatch
2309 .clear_pending_modify_if_command(&cmd.client_order_id, cmd.command_id);
2310 let reason = format!("Lighter modify_order failed: {e}");
2311 log::warn!("{reason} for {}", cmd.client_order_id);
2312 self.emitter.emit_order_modify_rejected_event(
2313 cmd.strategy_id,
2314 cmd.instrument_id,
2315 cmd.client_order_id,
2316 cmd.venue_order_id,
2317 &reason,
2318 self.clock.get_time_ns(),
2319 );
2320
2321 if let Some(deferred) = events.remove(&cmd.client_order_id) {
2322 flush_modify_events(
2323 cmd.client_order_id,
2324 deferred,
2325 &self.dispatch,
2326 &self.emitter,
2327 &mut events,
2328 );
2329 }
2330
2331 return;
2332 }
2333 };
2334 let PreparedModifyOrder {
2335 client_order_id,
2336 strategy_id,
2337 instrument_id,
2338 venue_order_id,
2339 tx_info,
2340 nonce,
2341 api_key_index,
2342 tx_hash,
2343 mut send_reservation,
2344 } = prepared;
2345 let connection_epoch = send_reservation.connection_epoch;
2346
2347 let ws_client = self.ws_client.clone();
2348 let dispatch = self.dispatch.clone();
2349 let credential = credential.clone();
2350 let emitter = self.emitter.clone();
2351 let clock = self.clock;
2352
2353 let tx_rate_limiter = self.tx_rate_limiter.clone();
2354
2355 self.spawn_task("modify_order", async move {
2356 send_reservation.wait_for_turn().await;
2357 await_tx_quota(&tx_rate_limiter).await;
2358
2359 let pending = PendingSendTx {
2360 batch_id: None,
2361 connection_epoch,
2362 kind: PendingSendTxKind::Modify {
2363 strategy_id,
2364 instrument_id,
2365 client_order_id,
2366 venue_order_id,
2367 },
2368 submitted_at: clock.get_time_ns(),
2369 nonce,
2370 api_key_index,
2371 tx_hash,
2372 };
2373
2374 dispatch.enqueue_pending_sendtx(pending.clone());
2375 spawn_acked_order_probe(&pending, probe_context);
2376
2377 if let Err(e) = ws_client
2378 .send_tx_on_connection(LighterTxType::ModifyOrder as u8, tx_info, connection_epoch)
2379 .await
2380 {
2381 let failure = classify_lighter_ws_command_failure("modify_order", &e);
2382 let reason = command_failure_reason(&failure);
2383 if matches!(&failure, CommandFailure::Ambiguous(_)) {
2384 log::warn!(
2385 "Lighter modify_order dispatch outcome unknown for {client_order_id}: {reason}; \
2386 retaining pending state for venue reconciliation; diagnostic={e:?}",
2387 );
2388 } else {
2389 log::warn!("{reason} for {client_order_id}; diagnostic={e:?}");
2390 dispatch.remove_pending_sendtx_by_nonce(connection_epoch, nonce);
2391 rollback_tx_dispatch(&dispatch, &credential, None, nonce);
2392 reject_modify_order(&dispatch, &emitter, &pending, reason, clock.get_time_ns());
2393 }
2394 }
2395 send_reservation.release();
2396 Ok(())
2397 });
2398 }
2399
2400 fn admit_modify_order(&self, cmd: &ModifyOrder) -> bool {
2401 let _events = self.dispatch.order_events.lock();
2402
2403 let publication_pending = self
2404 .dispatch
2405 .order_identity(&cmd.client_order_id)
2406 .is_some_and(|identity| {
2407 identity
2408 .publication_pending(self.core.cache().order(&cmd.client_order_id).as_deref())
2409 });
2410
2411 if !publication_pending && self.dispatch.set_pending_modify_action(cmd) {
2412 return true;
2413 }
2414
2415 self.emitter.emit_order_modify_rejected_event(
2416 cmd.strategy_id,
2417 cmd.instrument_id,
2418 cmd.client_order_id,
2419 cmd.venue_order_id,
2420 "Lighter permits one unresolved modification per order; wait for its update or rejection",
2421 self.clock.get_time_ns(),
2422 );
2423 false
2424 }
2425
2426 fn prepare_signed_modify_order(
2427 &self,
2428 cmd: &ModifyOrder,
2429 credential: &Credential,
2430 ) -> anyhow::Result<PreparedModifyOrder> {
2431 let plan = self.prepare_modify_order_plan(cmd)?;
2432 let context = self.fanout_dispatch_context(credential)?;
2433 let reserved = self.build_tx_context(credential)?;
2434 context.sign_modify_order(&plan, reserved)
2435 }
2436
2437 fn prepare_modify_order_plan(&self, cmd: &ModifyOrder) -> anyhow::Result<ModifyOrderPlan> {
2438 let market_index = self
2439 .registry
2440 .market_index(&cmd.instrument_id)
2441 .ok_or_else(|| {
2442 anyhow::anyhow!(
2443 "no Lighter market_index registered for instrument {}",
2444 cmd.instrument_id,
2445 )
2446 })?;
2447
2448 let voi = cmd
2449 .venue_order_id
2450 .or_else(|| self.dispatch.lookup_venue_order_id(&cmd.client_order_id))
2451 .ok_or_else(|| {
2452 anyhow::anyhow!(
2453 "cannot modify Lighter order {}: venue order_id not yet known \
2454 (await OrderAccepted before issuing modify)",
2455 cmd.client_order_id,
2456 )
2457 })?;
2458
2459 let venue_index: i64 = voi
2460 .as_str()
2461 .parse()
2462 .with_context(|| format!("Lighter venue_order_id `{voi}` is not an integer index"))?;
2463
2464 let order = self.core.cache().try_order_owned(&cmd.client_order_id)?;
2465 let instrument = self
2466 .core
2467 .cache()
2468 .try_instrument(&cmd.instrument_id)?
2469 .clone();
2470
2471 let new_qty = cmd.quantity.unwrap_or(order.quantity());
2472 anyhow::ensure!(
2473 !(order.parent_order_id().is_some()
2474 && self
2475 .dispatch
2476 .snapshot_for(&cmd.client_order_id)
2477 .is_some_and(|shape| shape.quantity.is_zero())),
2478 "Lighter cannot modify an OTO child while it waits for parent execution"
2479 );
2480 let price_precision = instrument.price_precision();
2481 let new_trigger = cmd.trigger_price.or(order.trigger_price());
2482 let triggered = order.is_triggered() == Some(true)
2483 || self.dispatch.triggered_was_emitted(&cmd.client_order_id);
2484 anyhow::ensure!(
2485 !triggered || cmd.trigger_price.is_none(),
2486 "Lighter cannot modify a trigger price after the order has triggered"
2487 );
2488
2489 let price_ticks = match order.order_type() {
2492 OrderType::StopMarket | OrderType::MarketIfTouched => {
2493 let trigger = new_trigger.ok_or_else(|| {
2494 anyhow::anyhow!("{:?} orders require a trigger_price", order.order_type())
2495 })?;
2496 let is_buy = matches!(order.order_side(), OrderSide::Buy);
2497 let slippage_bps = self.resolve_slippage_bps(cmd.params.as_ref());
2498
2499 derive_market_order_price_ticks(
2500 trigger.as_decimal(),
2501 is_buy,
2502 price_precision,
2503 slippage_bps,
2504 )?
2505 }
2506 _ => {
2507 let new_price = cmd.price.or(order.price()).ok_or_else(|| {
2508 anyhow::anyhow!("modify_order requires a price (none on order or command)")
2509 })?;
2510
2511 price_to_ticks(&new_price, price_precision)?
2512 }
2513 };
2514
2515 let base_amount = quantity_to_ticks(&new_qty, instrument.size_precision())?;
2516 anyhow::ensure!(
2517 base_amount > 0,
2518 "quantity `{new_qty}` rounds to 0 ticks at size_precision {}",
2519 instrument.size_precision(),
2520 );
2521
2522 let reduced_qty = self
2524 .dispatch
2525 .order_identity(&cmd.client_order_id)
2526 .map_or(Decimal::ZERO, |identity| identity.reduced_qty());
2527 let reduced_qty = quantity_from_decimal(reduced_qty, instrument.size_precision())?;
2528 let reduced_amount = quantity_to_ticks(&reduced_qty, instrument.size_precision())?;
2529 let base_amount = base_amount
2530 .checked_add(reduced_amount)
2531 .context("Lighter modified quantity overflow")?;
2532
2533 let trigger_price_ticks = match new_trigger.filter(|_| !triggered) {
2534 Some(trigger) if !trigger.is_zero() => price_to_ticks(&trigger, price_precision)?,
2535 _ => 0,
2536 };
2537
2538 if !triggered
2539 && matches!(
2540 order.order_type(),
2541 OrderType::StopMarket
2542 | OrderType::StopLimit
2543 | OrderType::MarketIfTouched
2544 | OrderType::LimitIfTouched
2545 )
2546 {
2547 anyhow::ensure!(
2548 trigger_price_ticks > 0,
2549 "trigger_price `{new_trigger:?}` rounds to 0 ticks at precision {price_precision}",
2550 );
2551 }
2552 validate_order_amount(&instrument, new_qty, price_ticks, price_precision)?;
2553 self.dispatch.set_pending_modify_shape(
2554 cmd,
2555 OrderShapeSnapshot {
2556 quantity: parse_quantity_from_ticks(base_amount, instrument.size_precision())?,
2557 price: LIMIT_ORDER_TYPES
2558 .contains(&order.order_type())
2559 .then(|| parse_price_from_ticks(price_ticks, price_precision))
2560 .transpose()?,
2561 trigger_price: STOP_ORDER_TYPES
2562 .contains(&order.order_type())
2563 .then(|| {
2564 if triggered {
2565 Ok(None)
2566 } else {
2567 parse_price_from_ticks(trigger_price_ticks, price_precision).map(Some)
2568 }
2569 })
2570 .transpose()?
2571 .flatten(),
2572 },
2573 );
2574
2575 Ok(ModifyOrderPlan {
2576 command_id: cmd.command_id,
2577 client_order_id: cmd.client_order_id,
2578 strategy_id: cmd.strategy_id,
2579 instrument_id: cmd.instrument_id,
2580 venue_order_id: Some(voi),
2581 market_index,
2582 venue_index,
2583 base_amount,
2584 price: price_ticks,
2585 trigger_price: trigger_price_ticks,
2586 })
2587 }
2588
2589 pub fn update_leverage(
2608 &self,
2609 instrument_id: InstrumentId,
2610 initial_margin_fraction: u16,
2611 margin_mode: LighterPositionMarginMode,
2612 ) -> anyhow::Result<()> {
2613 let credential = self.credential.as_ref().ok_or_else(|| {
2614 anyhow::anyhow!("Lighter execution client cannot update leverage without credentials")
2615 })?;
2616
2617 let market_index = self.registry.market_index(&instrument_id).ok_or_else(|| {
2618 anyhow::anyhow!("no Lighter market_index registered for instrument {instrument_id}")
2619 })?;
2620
2621 anyhow::ensure!(
2622 (1..=10_000).contains(&initial_margin_fraction),
2623 "initial_margin_fraction must be in 1..=10_000, was {initial_margin_fraction}",
2624 );
2625
2626 let ReservedTxContext {
2627 context,
2628 mut send_reservation,
2629 } = self.build_tx_context(credential)?;
2630
2631 let connection_epoch = send_reservation.connection_epoch;
2632
2633 let captured_nonce = context.nonce;
2634 let captured_api_key_index = context.api_key_index;
2635 let mut rollback_guard =
2636 TxDispatchGuard::new(self.dispatch.clone(), credential, None, captured_nonce);
2637
2638 let tx = UpdateLeverageTxInfo {
2639 context,
2640 market_index,
2641 initial_margin_fraction,
2642 margin_mode: margin_mode as u8,
2643 skip_nonce: 0,
2644 };
2645
2646 let signed = sign_tx(
2647 &tx,
2648 self.config.chain_id(),
2649 &credential.private_key()?,
2650 fresh_k(),
2651 )?;
2652
2653 let tx_info_str = TxInfoJson::update_leverage(&tx, &signed);
2654 let tx_info = serde_json::value::RawValue::from_string(tx_info_str)
2655 .context("failed to wrap signed Lighter update_leverage tx_info JSON")?;
2656
2657 rollback_guard.disarm();
2658 let captured_tx_hash = signed.tx_hash_hex();
2659
2660 let ws_client = self.ws_client.clone();
2661 let dispatch = self.dispatch.clone();
2662 let credential = credential.clone();
2663 let clock = self.clock;
2664
2665 let tx_rate_limiter = self.tx_rate_limiter.clone();
2666
2667 self.spawn_task("update_leverage", async move {
2668 send_reservation.wait_for_turn().await;
2669 await_tx_quota(&tx_rate_limiter).await;
2670 dispatch.enqueue_pending_sendtx(PendingSendTx {
2671 batch_id: None,
2672 connection_epoch,
2673 kind: PendingSendTxKind::Other,
2674 submitted_at: clock.get_time_ns(),
2675 nonce: captured_nonce,
2676 api_key_index: captured_api_key_index,
2677 tx_hash: captured_tx_hash,
2678 });
2679
2680 if let Err(e) = ws_client
2681 .send_tx_on_connection(
2682 LighterTxType::UpdateLeverage as u8,
2683 tx_info,
2684 connection_epoch,
2685 )
2686 .await
2687 {
2688 let failure = classify_lighter_ws_command_failure("update_leverage", &e);
2689 let reason = command_failure_reason(&failure);
2690 if matches!(&failure, CommandFailure::Ambiguous(_)) {
2691 log::warn!(
2692 "Lighter update_leverage dispatch outcome unknown for {instrument_id}: \
2693 {reason}; retaining pending nonce for venue reconciliation; diagnostic={e:?}",
2694 );
2695 } else {
2696 log::error!("{reason} for {instrument_id}; diagnostic={e:?}");
2697 dispatch.remove_pending_sendtx_by_nonce(connection_epoch, captured_nonce);
2698 rollback_tx_dispatch(&dispatch, &credential, None, captured_nonce);
2699 }
2700 }
2701 send_reservation.release();
2702 Ok(())
2703 });
2704
2705 Ok(())
2706 }
2707}
2708
2709fn emit_lighter_position_reports(
2710 reports: Vec<PositionStatusReport>,
2711 removed: Vec<InstrumentId>,
2712 emitter: &ExecutionEventEmitter,
2713 account_id: AccountId,
2714 now: UnixNanos,
2715) {
2716 for report in reports {
2717 log::debug!(
2718 "Lighter PositionStatusReport: instrument={} side={:?} qty={}",
2719 report.instrument_id,
2720 report.position_side,
2721 report.quantity,
2722 );
2723 emitter.send_position_report(report);
2724 }
2725
2726 for instrument_id in removed {
2728 let flat = PositionStatusReport::new(
2729 account_id,
2730 instrument_id,
2731 PositionSide::Flat,
2732 Quantity::zero(0),
2733 now,
2734 now,
2735 Some(UUID4::new()),
2736 None,
2737 None,
2738 );
2739 emitter.send_position_report(flat);
2740 }
2741}
2742
2743#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2744enum AuthTokenRefreshOutcome {
2745 Rotated,
2746 Cancelled,
2747 Exhausted,
2748}
2749
2750#[derive(Debug, Clone, Copy)]
2751struct AuthTokenRefreshBackoff {
2752 initial_delay: Duration,
2753 max_delay: Duration,
2754 window: Duration,
2755}
2756
2757#[derive(Clone)]
2758struct NonceRefreshRetry {
2759 http_client: LighterHttpClient,
2760 dispatch: WsDispatchState,
2761 credential: Option<Credential>,
2762 submission_gate: Arc<tokio::sync::RwLock<()>>,
2763 ready_connection_epoch: Arc<AtomicU64>,
2764 ws_client: LighterWebSocketClient,
2765 cancellation_token: CancellationToken,
2766 pending_tasks: TaskSpawner,
2767}
2768
2769impl NonceRefreshRetry {
2770 fn spawn(self, connection_epoch: u64) {
2771 let pending_tasks = self.pending_tasks.clone();
2772
2773 let future = async move {
2774 let Some(credential) = self.credential else {
2775 return;
2776 };
2777 let mut retry_delay = NONCE_REFRESH_RETRY_INITIAL_DELAY;
2778
2779 loop {
2780 tokio::select! {
2781 () = self.cancellation_token.cancelled() => return,
2782 () = tokio::time::sleep(retry_delay) => {}
2783 }
2784
2785 if self.ws_client.connection_epoch() != connection_epoch
2786 || self.ready_connection_epoch.load(Ordering::Acquire) == connection_epoch
2787 {
2788 return;
2789 }
2790
2791 let _refresh_guard = tokio::select! {
2792 () = self.cancellation_token.cancelled() => return,
2793 guard = self.submission_gate.write() => guard,
2794 };
2795
2796 if self.ws_client.connection_epoch() != connection_epoch
2797 || self.ready_connection_epoch.load(Ordering::Acquire) == connection_epoch
2798 {
2799 return;
2800 }
2801
2802 let result = tokio::select! {
2803 () = self.cancellation_token.cancelled() => return,
2804 result = self.http_client.get_next_nonce(
2805 credential.account_index(),
2806 credential.api_key_index(),
2807 ) => result,
2808 };
2809
2810 match result {
2811 Ok(response) => {
2812 if self.ws_client.connection_epoch() != connection_epoch {
2813 return;
2814 }
2815 self.dispatch.nonce_manager.refresh(
2816 credential.account_index(),
2817 credential.api_key_index(),
2818 response.nonce,
2819 );
2820 self.ready_connection_epoch
2821 .store(connection_epoch, Ordering::Release);
2822 log::info!(
2823 "Recovered Lighter nonce refresh: account_index={}, \
2824 connection_epoch={connection_epoch}, next_nonce={}",
2825 credential.account_index(),
2826 response.nonce,
2827 );
2828 return;
2829 }
2830 Err(e) => {
2831 log::warn!(
2832 "Failed to retry Lighter nonce refresh for connection epoch \
2833 {connection_epoch}: {e}",
2834 );
2835 }
2836 }
2837
2838 retry_delay = retry_delay
2839 .saturating_mul(2)
2840 .min(NONCE_REFRESH_RETRY_MAX_DELAY);
2841 }
2842 };
2843
2844 if let Err(e) = pending_tasks.spawn(future) {
2845 log::debug!("Skipping Lighter nonce refresh retry during shutdown: {e}");
2846 }
2847 }
2848}
2849
2850fn auth_token_rotation_channels(account_index: i64) -> [LighterWsChannel; 5] {
2851 [
2852 LighterWsChannel::AccountAllOrders(account_index),
2853 LighterWsChannel::AccountAllTrades(account_index),
2854 LighterWsChannel::AccountAllPositions(account_index),
2855 LighterWsChannel::AccountAllAssets(account_index),
2856 LighterWsChannel::UserStats(account_index),
2857 ]
2858}
2859
2860async fn refresh_auth_token_until_rotated<MintToken, Subscribe, SubscribeFuture>(
2861 credential: &Credential,
2862 channels: &[LighterWsChannel],
2863 cancellation_token: &CancellationToken,
2864 backoff: AuthTokenRefreshBackoff,
2865 mut mint_token: MintToken,
2866 mut subscribe: Subscribe,
2867) -> AuthTokenRefreshOutcome
2868where
2869 MintToken: FnMut(&Credential) -> anyhow::Result<SecretString>,
2870 Subscribe: FnMut(LighterWsChannel, SecretString) -> SubscribeFuture,
2871 SubscribeFuture: Future<Output = Result<(), crate::websocket::error::LighterWsError>>,
2872{
2873 let retry_started = tokio::time::Instant::now();
2874 let mut retry_delay = backoff.initial_delay.min(backoff.max_delay);
2875 let mut attempt = 1_u32;
2876
2877 loop {
2878 match rotate_auth_token_once(credential, channels, &mut mint_token, &mut subscribe).await {
2879 Ok(()) => {
2880 log::debug!(
2881 "Lighter auth-token rotated for account_index={}, attempts={attempt}",
2882 credential.account_index(),
2883 );
2884 return AuthTokenRefreshOutcome::Rotated;
2885 }
2886 Err(e) => {
2887 log::warn!("Lighter auth-token rotation attempt {attempt} failed: {e:#}");
2888 }
2889 }
2890
2891 let Some(remaining) = backoff.window.checked_sub(retry_started.elapsed()) else {
2892 log::error!(
2893 "Lighter auth-token rotation retry window exhausted: account_index={}, attempts={attempt}",
2894 credential.account_index(),
2895 );
2896 return AuthTokenRefreshOutcome::Exhausted;
2897 };
2898
2899 if remaining.as_nanos() == 0 {
2900 log::error!(
2901 "Lighter auth-token rotation retry window exhausted: account_index={}, attempts={attempt}",
2902 credential.account_index(),
2903 );
2904 return AuthTokenRefreshOutcome::Exhausted;
2905 }
2906
2907 let delay = retry_delay.min(remaining);
2908 log::warn!(
2909 "Retrying Lighter auth-token rotation in {:.3}s: account_index={}, attempts={attempt}",
2910 delay.as_secs_f64(),
2911 credential.account_index(),
2912 );
2913
2914 if !sleep_or_auth_token_refresh_cancelled(delay, cancellation_token).await {
2915 return AuthTokenRefreshOutcome::Cancelled;
2916 }
2917
2918 retry_delay = next_auth_token_refresh_retry_delay(retry_delay, backoff.max_delay);
2919 attempt = attempt.saturating_add(1);
2920 }
2921}
2922
2923async fn rotate_auth_token_once<MintToken, Subscribe, SubscribeFuture>(
2924 credential: &Credential,
2925 channels: &[LighterWsChannel],
2926 mint_token: &mut MintToken,
2927 subscribe: &mut Subscribe,
2928) -> anyhow::Result<()>
2929where
2930 MintToken: FnMut(&Credential) -> anyhow::Result<SecretString>,
2931 Subscribe: FnMut(LighterWsChannel, SecretString) -> SubscribeFuture,
2932 SubscribeFuture: Future<Output = Result<(), crate::websocket::error::LighterWsError>>,
2933{
2934 let token =
2935 mint_token(credential).context("failed to mint Lighter auth token during rotation")?;
2936 let mut first_error = None;
2937
2938 for channel in channels {
2939 if let Err(e) = subscribe(channel.clone(), token.clone()).await {
2940 log::debug!("Lighter auth-token rotation: re-subscribe failed for {channel:?}: {e}",);
2941 first_error.get_or_insert_with(|| format!("{channel:?}: {e}"));
2942 }
2943 }
2944
2945 if let Some(error) = first_error {
2946 anyhow::bail!("failed to re-subscribe Lighter account channels: {error}");
2947 }
2948
2949 Ok(())
2950}
2951
2952async fn sleep_or_auth_token_refresh_cancelled(
2953 duration: Duration,
2954 cancellation_token: &CancellationToken,
2955) -> bool {
2956 tokio::select! {
2957 () = cancellation_token.cancelled() => false,
2958 () = tokio::time::sleep(duration) => true,
2959 }
2960}
2961
2962fn auth_token_refresh_next_delay(outcome: AuthTokenRefreshOutcome) -> Option<Duration> {
2963 match outcome {
2964 AuthTokenRefreshOutcome::Rotated => Some(AUTH_TOKEN_REFRESH_INTERVAL),
2965 AuthTokenRefreshOutcome::Cancelled => None,
2966 AuthTokenRefreshOutcome::Exhausted => Some(AUTH_TOKEN_REFRESH_RETRY_MAX_DELAY),
2967 }
2968}
2969
2970fn next_auth_token_refresh_retry_delay(current: Duration, max: Duration) -> Duration {
2971 current.checked_mul(2).unwrap_or(max).min(max)
2972}
2973
2974#[derive(Debug)]
2975struct ReservedTxContext {
2976 context: TxContext,
2977 send_reservation: TxSendReservation,
2978}
2979
2980#[derive(Debug, Clone)]
2981struct TxSendSequencer {
2982 state: Arc<Mutex<TxSendSequencerState>>,
2983 version: Arc<AtomicU64>,
2984 changed: tokio::sync::watch::Sender<u64>,
2985}
2986
2987impl TxSendSequencer {
2988 fn new() -> Self {
2989 let (changed, _) = tokio::sync::watch::channel(0);
2990 Self {
2991 state: Arc::new(Mutex::new(TxSendSequencerState::default())),
2992 version: Arc::new(AtomicU64::new(0)),
2993 changed,
2994 }
2995 }
2996
2997 fn reserve(
2998 &self,
2999 account_index: i64,
3000 api_key_index: u8,
3001 allocate_nonce: impl FnOnce() -> Result<i64, NonceError>,
3002 ) -> Result<TxSendReservation, NonceError> {
3003 let key = TxSendKey {
3004 account_index,
3005 api_key_index,
3006 };
3007
3008 let mut state = self.state.lock();
3009 let nonce = allocate_nonce()?;
3010 *state
3011 .pending
3012 .entry(key)
3013 .or_default()
3014 .entry(nonce)
3015 .or_default() += 1;
3016 drop(state);
3017 self.notify_waiters();
3018 Ok(TxSendReservation {
3019 sequencer: self.clone(),
3020 key,
3021 nonce,
3022 released: false,
3023 nonce_guard: None,
3024 connection_epoch: 0,
3025 })
3026 }
3027
3028 async fn wait_for_turn(&self, key: TxSendKey, nonce: i64) {
3029 let mut changed = self.changed.subscribe();
3030
3031 loop {
3032 if self.ready_to_send(key, nonce) {
3033 return;
3034 }
3035
3036 if changed.changed().await.is_err() {
3037 tokio::task::yield_now().await;
3038 }
3039 }
3040 }
3041
3042 fn release(&self, key: TxSendKey, nonce: i64) {
3043 let mut state = self.state.lock();
3044
3045 let should_notify = if let Some(pending) = state.pending.get_mut(&key) {
3046 let removed = if let Some(count) = pending.get_mut(&nonce) {
3047 *count -= 1;
3048 if *count == 0 {
3049 pending.remove(&nonce);
3050 }
3051
3052 true
3053 } else {
3054 false
3055 };
3056
3057 if pending.is_empty() {
3058 state.pending.remove(&key);
3059 }
3060 removed
3061 } else {
3062 false
3063 };
3064 drop(state);
3065
3066 if should_notify {
3067 self.notify_waiters();
3068 }
3069 }
3070
3071 fn ready_to_send(&self, key: TxSendKey, nonce: i64) -> bool {
3072 let state = self.state.lock();
3073 state
3074 .pending
3075 .get(&key)
3076 .and_then(|pending| pending.first_key_value())
3077 .is_none_or(|(first, _)| *first >= nonce)
3078 }
3079
3080 fn notify_waiters(&self) {
3081 let version = self.version.fetch_add(1, Ordering::AcqRel) + 1;
3082 let _ = self.changed.send(version);
3083 }
3084}
3085
3086#[derive(Debug, Default)]
3087struct TxSendSequencerState {
3088 pending: BTreeMap<TxSendKey, BTreeMap<i64, usize>>,
3089}
3090
3091#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
3092struct TxSendKey {
3093 account_index: i64,
3094 api_key_index: u8,
3095}
3096
3097#[derive(Debug)]
3098struct TxSendReservation {
3099 sequencer: TxSendSequencer,
3100 key: TxSendKey,
3101 nonce: i64,
3102 released: bool,
3103 nonce_guard: Option<tokio::sync::OwnedRwLockReadGuard<()>>,
3104 connection_epoch: u64,
3105}
3106
3107impl TxSendReservation {
3108 fn with_nonce_ownership(
3109 mut self,
3110 nonce_guard: tokio::sync::OwnedRwLockReadGuard<()>,
3111 connection_epoch: u64,
3112 ) -> Self {
3113 self.nonce_guard = Some(nonce_guard);
3114 self.connection_epoch = connection_epoch;
3115 self
3116 }
3117
3118 async fn wait_for_turn(&self) {
3119 self.sequencer.wait_for_turn(self.key, self.nonce).await;
3120 }
3121
3122 fn release(&mut self) {
3123 if self.released {
3124 return;
3125 }
3126
3127 self.sequencer.release(self.key, self.nonce);
3128 self.nonce_guard.take();
3129 self.released = true;
3130 }
3131}
3132
3133impl Drop for TxSendReservation {
3134 fn drop(&mut self) {
3135 self.release();
3136 }
3137}
3138
3139#[cfg(test)]
3140async fn wait_for_tx_send_reservations(reservations: &[&TxSendReservation]) {
3141 let Some(first) = reservations.first() else {
3142 return;
3143 };
3144
3145 debug_assert!(
3146 reservations
3147 .iter()
3148 .all(|reservation| reservation.key == first.key),
3149 "batch send reservations must share one nonce stream",
3150 );
3151
3152 let nonce = reservations
3153 .iter()
3154 .map(|reservation| reservation.nonce)
3155 .min()
3156 .expect("reservations is non-empty");
3157 first.sequencer.wait_for_turn(first.key, nonce).await;
3158}
3159
3160struct PreparedCreateOrder {
3161 order: OrderAny,
3162 client_order_index: i64,
3163 tx_info: Box<serde_json::value::RawValue>,
3164 nonce: i64,
3165 api_key_index: u8,
3166 tx_hash: String,
3167 send_reservation: TxSendReservation,
3168}
3169
3170struct CreateOrderPlan {
3171 order: OrderAny,
3172 market_index: i64,
3173 base_amount: i64,
3174 price: u32,
3175 order_type: u8,
3176 time_in_force: u8,
3177 trigger_price: u32,
3178 order_expiry: i64,
3179 attributes: L2TxAttributes,
3180}
3181
3182impl CreateOrderPlan {
3183 fn order_info(&self, client_order_index: i64) -> OrderInfo {
3184 OrderInfo {
3185 market_index: self.market_index,
3186 client_order_index,
3187 base_amount: self.base_amount,
3188 price: self.price,
3189 is_ask: self.order.order_side() == OrderSide::Sell,
3190 order_type: self.order_type,
3191 time_in_force: self.time_in_force,
3192 reduce_only: self.order.is_reduce_only(),
3193 trigger_price: self.trigger_price,
3194 order_expiry: self.order_expiry,
3195 }
3196 }
3197}
3198
3199struct CreateGroupPlan {
3200 grouping_type: LighterGroupingType,
3201 orders: Vec<CreateOrderPlan>,
3202}
3203
3204struct PreparedCancelOrder {
3205 client_order_id: ClientOrderId,
3206 strategy_id: StrategyId,
3207 instrument_id: InstrumentId,
3208 venue_order_id: Option<VenueOrderId>,
3209 tx_info: Box<serde_json::value::RawValue>,
3210 nonce: i64,
3211 api_key_index: u8,
3212 tx_hash: String,
3213 send_reservation: TxSendReservation,
3214}
3215
3216struct CancelOrderPlan {
3217 client_order_id: ClientOrderId,
3218 strategy_id: StrategyId,
3219 instrument_id: InstrumentId,
3220 venue_order_id: Option<VenueOrderId>,
3221 market_index: i64,
3222 venue_index: i64,
3223}
3224
3225#[derive(Clone)]
3226struct FanoutDispatchContext {
3227 clock: &'static AtomicTime,
3228 chain_id: u32,
3229 integrator_account_index: Option<u64>,
3230 emitter: ExecutionEventEmitter,
3231 credential: Credential,
3232 http_client: LighterHttpClient,
3233 registry: Arc<MarketRegistry>,
3234 ws_client: LighterWebSocketClient,
3235 tx_rate_limiter: Arc<LighterTxRateLimiter>,
3236 tx_send_sequencer: TxSendSequencer,
3237 nonce_submission_gate: Arc<tokio::sync::RwLock<()>>,
3238 nonce_ready_connection_epoch: Arc<AtomicU64>,
3239 dispatch: WsDispatchState,
3240 nonce_recovery_inflight: Arc<AtomicBool>,
3241 pending_tasks: TaskSpawner,
3242}
3243
3244impl FanoutDispatchContext {
3245 async fn refresh_scheduled_cancel(&self, timeout_secs: u64) -> anyhow::Result<()> {
3246 let deadline = self
3247 .clock
3248 .get_time_ms()
3249 .checked_add(timeout_secs * 1_000)
3250 .and_then(|deadline| i64::try_from(deadline).ok())
3251 .context("scheduled cancellation deadline overflows milliseconds")?;
3252 let cancellation = self.pending_tasks.cancellation_token().child_token();
3253 let _guard = cancellation.clone().drop_guard();
3254 tokio::time::timeout(
3255 SCHEDULED_CANCEL_REQUEST_TIMEOUT,
3256 self.send_cancel_all(
3257 LighterCancelAllTimeInForce::Scheduled,
3258 deadline,
3259 true,
3260 cancellation,
3261 ),
3262 )
3263 .await
3264 .context("scheduled cancellation request timed out")??;
3265 Ok(())
3266 }
3267
3268 async fn send_cancel_all(
3269 &self,
3270 time_in_force: LighterCancelAllTimeInForce,
3271 scheduled_time_ms: i64,
3272 require_execution: bool,
3273 cancellation: CancellationToken,
3274 ) -> anyhow::Result<String> {
3275 let credential = &self.credential;
3276 let ReservedTxContext {
3277 context,
3278 mut send_reservation,
3279 } = self.build_tx_context()?;
3280 let mut rollback_guard =
3281 TxDispatchGuard::new(self.dispatch.clone(), credential, None, context.nonce);
3282
3283 let tx = CancelAllOrdersTxInfo {
3284 context,
3285 time_in_force: time_in_force as u8,
3286 scheduled_time_ms,
3287 skip_nonce: 0,
3288 };
3289
3290 let signed = sign_tx(&tx, self.chain_id, &credential.private_key()?, fresh_k())?;
3291 let tx_info = TxInfoJson::cancel_all_orders(&tx, &signed);
3292 let tx_info = serde_json::value::RawValue::from_string(tx_info)
3293 .context("failed to wrap signed Lighter account cancellation JSON")?;
3294 let tx_hash = signed.tx_hash_hex();
3295 let connection_epoch = send_reservation.connection_epoch;
3296 let ws_client = self.ws_client.clone();
3297 let dispatch = self.dispatch.clone();
3298 let credential = credential.clone();
3299 let clock = self.clock;
3300 let tx_rate_limiter = self.tx_rate_limiter.clone();
3301 let dispatched_hash = tx_hash.clone();
3302 let (response_tx, response_rx) = tokio::sync::oneshot::channel();
3303
3304 self.pending_tasks
3305 .spawn(async move {
3306 {
3307 let ready = async {
3308 send_reservation.wait_for_turn().await;
3309 await_tx_quota(&tx_rate_limiter).await;
3310 };
3311
3312 tokio::pin!(ready);
3313 tokio::select! {
3314 biased;
3315 () = cancellation.cancelled() => {
3316 if matches!(dispatch.nonce_manager.ack_failure_if_latest(
3318 context.account_index, context.api_key_index, context.nonce,
3319 ), Ok(true)) {
3320 rollback_guard.disarm();
3321 return;
3322 }
3323 ready.await;
3324 }
3325 () = &mut ready => {},
3326 }
3327 }
3328
3329 dispatch.enqueue_pending_sendtx(PendingSendTx {
3330 batch_id: None,
3331 connection_epoch,
3332 kind: PendingSendTxKind::Other,
3333 submitted_at: clock.get_time_ns(),
3334 nonce: context.nonce,
3335 api_key_index: context.api_key_index,
3336 tx_hash: dispatched_hash.clone(),
3337 });
3338
3339 rollback_guard.disarm();
3340 let response = ws_client.send_tx_on_connection(
3341 LighterTxType::CancelAllOrders as u8, tx_info, connection_epoch,
3342 ).await;
3343
3344 if let Err(e) = &response {
3345 let failure =
3346 classify_lighter_ws_command_failure("cancel_all_orders_account", e);
3347 log::warn!(
3348 "Lighter account cancellation transaction {dispatched_hash} dispatch failed, nonce={}: {failure:?}; diagnostic={e}",
3349 context.nonce,
3350 );
3351
3352 if !matches!(failure, CommandFailure::Ambiguous(_)) {
3353 dispatch.remove_pending_sendtx_by_hash(connection_epoch, &dispatched_hash);
3354 rollback_tx_dispatch(&dispatch, &credential, None, context.nonce);
3355 }
3356 }
3357
3358 send_reservation.release();
3359 let _ = response_tx.send(response);
3360 })
3361 .context("failed to spawn Lighter account cancellation")?;
3362
3363 let response = response_rx.await.with_context(|| {
3364 format!("Lighter account cancellation transaction {tx_hash} dispatch outcome unknown after session shutdown")
3365 })?;
3366
3367 response.with_context(|| {
3368 format!("failed to submit Lighter account cancellation transaction {tx_hash}")
3369 })?;
3370
3371 let confirmation = async {
3372 loop {
3373 match self.http_client.get_tx(tx_hash.clone()).await {
3374 Ok(tx) => {
3375 anyhow::ensure!(
3376 tx.status != LighterTxStatus::Failed,
3377 "Lighter account cancellation transaction {tx_hash} failed: {}",
3378 tx.event_info,
3379 );
3380
3381 if require_execution && tx.status != LighterTxStatus::Executed {
3382 tokio::time::sleep(ACKED_ORDER_LOOKUP_DELAY).await;
3383 continue;
3384 }
3385
3386 let event = parse_acked_tx_event(&tx.event_info).with_context(|| {
3387 format!("failed to parse Lighter account cancellation transaction {tx_hash} event")
3388 })?;
3389
3390 anyhow::ensure!(
3391 event.app_error.is_empty(),
3392 "Lighter account cancellation transaction {tx_hash} failed: {}",
3393 event.app_error,
3394 );
3395
3396 anyhow::ensure!(
3397 !require_execution
3398 || (tx_hash_matches(&tx.hash, &tx_hash)
3399 && tx.tx_type == LighterTxType::CancelAllOrders as u8
3400 && tx.account_index == self.credential.account_index()
3401 && tx.api_key_index == self.credential.api_key_index()
3402 && tx.nonce == context.nonce),
3403 "Lighter scheduled cancellation transaction identity mismatch",
3404 );
3405 return Ok(tx_hash.clone());
3406 }
3407 Err(LighterHttpError::Venue { code: 21500, .. }) => {}
3408 Err(e) => {
3409 return Err(e).with_context(|| {
3410 format!("failed to look up Lighter account cancellation transaction {tx_hash}")
3411 });
3412 }
3413 }
3414
3415 tokio::time::sleep(ACKED_ORDER_LOOKUP_DELAY).await;
3416 }
3417 };
3418
3419 tokio::time::timeout(
3420 if require_execution {
3421 SCHEDULED_CANCEL_REQUEST_TIMEOUT
3422 } else {
3423 ACCOUNT_CANCEL_CONFIRM_TIMEOUT
3424 },
3425 confirmation,
3426 )
3427 .await
3428 .with_context(|| {
3429 format!("Lighter account cancellation transaction {tx_hash} confirmation timed out")
3430 })?
3431 }
3432
3433 async fn dispatch_batch_plans(&self, plans: Vec<BatchOrderPlan>) {
3434 for chunk in plans.chunks(LIGHTER_MAX_BATCH_TX) {
3435 let mut batch = Vec::with_capacity(chunk.len());
3436 let mut remaining = chunk.iter().peekable();
3437 let mut retry_delay = NONCE_REFRESH_RETRY_INITIAL_DELAY;
3438
3439 while let Some(plan) = remaining.peek() {
3440 match self.sign_batch_plan(plan) {
3441 Ok(prepared) => {
3442 batch.push(prepared);
3443 retry_delay = NONCE_REFRESH_RETRY_INITIAL_DELAY;
3444 }
3445 Err(e) if should_retry_batch_signing(&e) => {
3446 self.send_batch(std::mem::take(&mut batch)).await;
3447 tokio::time::sleep(retry_delay).await;
3448 retry_delay = (retry_delay * 2).min(NONCE_REFRESH_RETRY_MAX_DELAY);
3449 continue;
3450 }
3451 Err(e) => self.reject_batch_plan(plan, &e.to_string()),
3452 }
3453
3454 remaining.next();
3455 }
3456
3457 self.send_batch(batch).await;
3458 }
3459 }
3460
3461 fn probe_context(&self) -> AckedOrderProbeContext {
3462 AckedOrderProbeContext {
3463 http_client: self.http_client.clone(),
3464 registry: Arc::clone(&self.registry),
3465 credential: self.credential.clone(),
3466 dispatch: self.dispatch.clone(),
3467 account_id: self.emitter.account_id(),
3468 clock: self.clock,
3469 emitter: self.emitter.clone(),
3470 connection_epoch: self.ws_client.connection_epoch_atomic(),
3471 cancellation_token: self.pending_tasks.cancellation_token(),
3472 pending_tasks: self.pending_tasks.clone(),
3473 }
3474 }
3475
3476 fn sign_batch_plan(&self, plan: &BatchOrderPlan) -> anyhow::Result<PreparedBatchTx> {
3477 match plan {
3478 BatchOrderPlan::Create(plan) => self.sign_create_order(plan).map(PreparedBatchTx::from),
3479 BatchOrderPlan::CreateGroup(plan) => self.sign_create_group(plan),
3480 BatchOrderPlan::Modify(plan) => {
3481 let reserved = self.build_tx_context()?;
3482 self.sign_modify_order(plan, reserved)
3483 .map(PreparedBatchTx::from)
3484 }
3485 BatchOrderPlan::Cancel(plan, emit) => self
3486 .sign_cancel_order(plan)
3487 .map(|prepared| PreparedBatchTx::from((prepared, *emit))),
3488 }
3489 }
3490
3491 fn reject_batch_plan(&self, plan: &BatchOrderPlan, error: &str) {
3492 match plan {
3493 BatchOrderPlan::CreateGroup(plan) => {
3494 for leg in &plan.orders {
3495 self.emitter.emit_order_denied(
3496 &leg.order,
3497 &format!("Lighter grouped submit failed: {error}"),
3498 );
3499 }
3500 }
3501 BatchOrderPlan::Create(plan) => {
3502 let reason = OrderDeniedReason::SubmitFailed {
3503 detail: format!("Lighter submit_order_list failed: {error}"),
3504 };
3505
3506 self.emitter
3507 .emit_order_denied(&plan.order, &reason.to_string());
3508 }
3509 BatchOrderPlan::Modify(plan) => {
3510 let mut events = self.dispatch.order_events.lock();
3511
3512 if self
3513 .dispatch
3514 .clear_pending_modify_if_command(&plan.client_order_id, plan.command_id)
3515 {
3516 self.emitter.emit_order_modify_rejected_event(
3517 plan.strategy_id,
3518 plan.instrument_id,
3519 plan.client_order_id,
3520 plan.venue_order_id,
3521 &format!("Lighter modify_order failed: {error}"),
3522 self.clock.get_time_ns(),
3523 );
3524
3525 if let Some(deferred) = events.remove(&plan.client_order_id) {
3526 flush_modify_events(
3527 plan.client_order_id,
3528 deferred,
3529 &self.dispatch,
3530 &self.emitter,
3531 &mut events,
3532 );
3533 }
3534 }
3535 }
3536 BatchOrderPlan::Cancel(plan, emit) => {
3537 self.dispatch.clear_pending_order_action_if(
3538 &plan.client_order_id,
3539 PendingOrderAction::cancel(*emit),
3540 );
3541 let reason = format!("Lighter cancel_order failed: {error}");
3542
3543 if *emit {
3544 self.emitter.emit_order_cancel_rejected_event(
3545 plan.strategy_id,
3546 plan.instrument_id,
3547 plan.client_order_id,
3548 plan.venue_order_id,
3549 &reason,
3550 self.clock.get_time_ns(),
3551 );
3552 } else {
3553 log::warn!(
3554 "{reason} for {}; suppressing OrderCancelRejected because order is not PendingCancel",
3555 plan.client_order_id
3556 );
3557 }
3558 }
3559 }
3560 }
3561
3562 fn build_tx_context(&self) -> anyhow::Result<ReservedTxContext> {
3563 let connection_epoch = self.ws_client.connection_epoch();
3564 anyhow::ensure!(
3565 self.nonce_ready_connection_epoch.load(Ordering::Acquire) == connection_epoch,
3566 TxPreparationError::NonceRefreshPending(connection_epoch),
3567 );
3568 let nonce_guard = Arc::clone(&self.nonce_submission_gate)
3569 .try_read_owned()
3570 .context("Lighter nonce refresh is in progress")?;
3571 anyhow::ensure!(
3572 self.ws_client.connection_epoch() == connection_epoch
3573 && self.nonce_ready_connection_epoch.load(Ordering::Acquire) == connection_epoch,
3574 TxPreparationError::StateChanged,
3575 );
3576
3577 let send_reservation = match self.tx_send_sequencer.reserve(
3578 self.credential.account_index(),
3579 self.credential.api_key_index(),
3580 || {
3581 self.dispatch.nonce_manager.next_nonce(
3582 self.credential.account_index(),
3583 self.credential.api_key_index(),
3584 )
3585 },
3586 ) {
3587 Ok(reservation) => reservation,
3588 Err(e @ NonceError::SkipWindowExhausted { .. }) => {
3589 self.spawn_nonce_window_recovery();
3590 let reason = format!("failed to allocate Lighter nonce: {e}");
3591 return Err(anyhow::Error::new(e).context(reason));
3592 }
3593 Err(e) => anyhow::bail!("failed to allocate Lighter nonce: {e}"),
3594 };
3595
3596 let nonce = send_reservation.nonce;
3597
3598 let now_ns = self.clock.get_time_ns().as_u64() as i64;
3599 let expired_at = (now_ns / 1_000_000).saturating_add(DEFAULT_TX_EXPIRY_MS);
3600 let send_reservation = send_reservation.with_nonce_ownership(nonce_guard, connection_epoch);
3601
3602 Ok(ReservedTxContext {
3603 context: TxContext {
3604 account_index: self.credential.account_index(),
3605 api_key_index: self.credential.api_key_index(),
3606 nonce,
3607 expired_at,
3608 },
3609 send_reservation,
3610 })
3611 }
3612
3613 fn spawn_nonce_window_recovery(&self) {
3614 if self.nonce_recovery_inflight.swap(true, Ordering::AcqRel) {
3615 return;
3616 }
3617
3618 let inflight = Arc::clone(&self.nonce_recovery_inflight);
3619 let http_client = self.http_client.clone();
3620 let dispatch = self.dispatch.clone();
3621 let nonce_submission_gate = Arc::clone(&self.nonce_submission_gate);
3622 let account_index = self.credential.account_index();
3623 let api_key_index = self.credential.api_key_index();
3624
3625 let future = async move {
3626 let _refresh_guard = nonce_submission_gate.write().await;
3627 let result = http_client
3628 .get_next_nonce(account_index, api_key_index)
3629 .await;
3630 inflight.store(false, Ordering::Release);
3631
3632 match result {
3633 Ok(response) => {
3634 let _ = dispatch.nonce_manager.sync_from_venue(
3635 account_index,
3636 api_key_index,
3637 response.nonce,
3638 );
3639 log::debug!(
3640 "Resynced Lighter nonce baseline after skip-window exhaustion: \
3641 account_index={account_index}, api_key_index={api_key_index}, \
3642 next_nonce={}",
3643 response.nonce,
3644 );
3645 }
3646 Err(e) => {
3647 log::error!("Failed to resync Lighter nonce after skip-window exhaustion: {e}");
3648 }
3649 }
3650 };
3651
3652 if let Err(e) = self.pending_tasks.spawn(future) {
3653 self.nonce_recovery_inflight.store(false, Ordering::Release);
3654 log::warn!("Skipping Lighter nonce recovery after shutdown began: {e}");
3655 }
3656 }
3657
3658 fn sign_create_order(&self, plan: &CreateOrderPlan) -> anyhow::Result<PreparedCreateOrder> {
3659 let cloid = plan.order.client_order_id();
3660 let client_order_index = self.dispatch.register_create_identity(&plan.order)?;
3661
3662 let ReservedTxContext {
3663 context,
3664 send_reservation,
3665 } = match self.build_tx_context() {
3666 Ok(context) => context,
3667 Err(e) => {
3668 self.dispatch.forget_cloid(client_order_index);
3669 self.dispatch.forget_order_identity(&cloid);
3670 return Err(e);
3671 }
3672 };
3673
3674 let nonce = context.nonce;
3675 let api_key_index = context.api_key_index;
3676 let mut rollback_guard = TxDispatchGuard::new(
3677 self.dispatch.clone(),
3678 &self.credential,
3679 Some(client_order_index),
3680 nonce,
3681 )
3682 .with_order_identity(cloid);
3683
3684 let tx = CreateOrderTxInfo {
3685 context,
3686 order: plan.order_info(client_order_index),
3687 attributes: plan.attributes,
3688 };
3689
3690 let signed = sign_tx(
3691 &tx,
3692 self.chain_id,
3693 &self.credential.private_key()?,
3694 fresh_k(),
3695 )?;
3696
3697 let tx_info =
3698 serde_json::value::RawValue::from_string(TxInfoJson::create_order(&tx, &signed))
3699 .context("failed to wrap signed Lighter tx_info JSON")?;
3700
3701 rollback_guard.disarm();
3702
3703 Ok(PreparedCreateOrder {
3704 order: plan.order.clone(),
3705 client_order_index,
3706 tx_info,
3707 nonce,
3708 api_key_index,
3709 tx_hash: signed.tx_hash_hex(),
3710 send_reservation,
3711 })
3712 }
3713
3714 fn sign_create_group(&self, plan: &CreateGroupPlan) -> anyhow::Result<PreparedBatchTx> {
3715 let reserved = self.build_tx_context()?;
3716 let mut guards = Vec::with_capacity(plan.orders.len());
3717 let mut orders = Vec::with_capacity(plan.orders.len());
3718 let mut bodies = Vec::with_capacity(plan.orders.len());
3719
3720 let mut nonce_guard = TxDispatchGuard::new(
3721 self.dispatch.clone(),
3722 &self.credential,
3723 None,
3724 reserved.context.nonce,
3725 );
3726
3727 for leg in &plan.orders {
3728 let index = self.dispatch.register_create_identity(&leg.order)?;
3729 guards.push(
3730 TxDispatchGuard::new(
3731 self.dispatch.clone(),
3732 &self.credential,
3733 Some(index),
3734 reserved.context.nonce,
3735 )
3736 .with_order_identity(leg.order.client_order_id()),
3737 );
3738 bodies.push(leg.order_info(index));
3739 orders.push((leg.order.clone(), index));
3740 }
3741
3742 let tx = CreateGroupedOrdersTxInfo {
3743 context: reserved.context,
3744 grouping_type: plan.grouping_type,
3745 orders: bodies,
3746 attributes: plan.orders[0].attributes,
3747 };
3748
3749 let signed = sign_tx(
3750 &tx,
3751 self.chain_id,
3752 &self.credential.private_key()?,
3753 fresh_k(),
3754 )?;
3755 let tx_info = serde_json::value::RawValue::from_string(TxInfoJson::create_grouped_orders(
3756 &tx, &signed,
3757 ))
3758 .context("failed to wrap signed Lighter grouped tx_info JSON")?;
3759
3760 for guard in &mut guards {
3761 guard.disarm();
3762 }
3763
3764 nonce_guard.disarm();
3765 Ok(PreparedBatchTx {
3766 tx_type: LighterTxType::CreateGroupedOrders as u8,
3767 tx_info,
3768 pending: PendingSendTx {
3769 batch_id: None,
3770 connection_epoch: reserved.send_reservation.connection_epoch,
3771 kind: PendingSendTxKind::CreateGroup { orders },
3772 submitted_at: UnixNanos::default(),
3773 nonce: reserved.context.nonce,
3774 api_key_index: reserved.context.api_key_index,
3775 tx_hash: signed.tx_hash_hex(),
3776 },
3777 send_reservation: reserved.send_reservation,
3778 })
3779 }
3780
3781 async fn send_create_order(&self, prepared: PreparedCreateOrder) {
3782 let PreparedCreateOrder {
3783 order,
3784 client_order_index,
3785 tx_info,
3786 nonce,
3787 api_key_index,
3788 tx_hash,
3789 mut send_reservation,
3790 } = prepared;
3791 let connection_epoch = send_reservation.connection_epoch;
3792 let client_order_id = order.client_order_id();
3793
3794 self.emitter.emit_order_submitted(&order);
3795 log::debug!("Lighter submit_order: queueing CreateOrder tx for {client_order_id}");
3796 send_reservation.wait_for_turn().await;
3797 await_tx_quota(&self.tx_rate_limiter).await;
3798 self.dispatch.enqueue_pending_sendtx(PendingSendTx {
3799 batch_id: None,
3800 connection_epoch,
3801 kind: PendingSendTxKind::Create {
3802 order: Box::new(order.clone()),
3803 client_order_index,
3804 },
3805 submitted_at: self.clock.get_time_ns(),
3806 nonce,
3807 api_key_index,
3808 tx_hash,
3809 });
3810
3811 if let Err(e) = self
3812 .ws_client
3813 .send_tx_on_connection(LighterTxType::CreateOrder as u8, tx_info, connection_epoch)
3814 .await
3815 {
3816 let failure = classify_lighter_ws_command_failure("submit_order", &e);
3817 let reason = command_failure_reason(&failure);
3818 if matches!(&failure, CommandFailure::Ambiguous(_)) {
3819 log::warn!(
3820 "Lighter submit_order dispatch outcome unknown for {client_order_id}: {reason}; \
3821 retaining pending state for venue reconciliation; diagnostic={e:?}",
3822 );
3823 } else {
3824 log::warn!("{reason} for {client_order_id}; diagnostic={e:?}");
3825 self.dispatch
3826 .remove_pending_sendtx_by_nonce(connection_epoch, nonce);
3827 rollback_tx_dispatch_create(
3828 &self.dispatch,
3829 &self.credential,
3830 Some(client_order_index),
3831 &client_order_id,
3832 nonce,
3833 );
3834 self.emitter
3835 .emit_order_rejected(&order, reason, self.clock.get_time_ns(), false);
3836 }
3837 }
3838 send_reservation.release();
3839 }
3840
3841 fn sign_modify_order(
3842 &self,
3843 plan: &ModifyOrderPlan,
3844 reserved: ReservedTxContext,
3845 ) -> anyhow::Result<PreparedModifyOrder> {
3846 let ReservedTxContext {
3847 context,
3848 send_reservation,
3849 } = reserved;
3850
3851 let captured_nonce = context.nonce;
3852 let captured_api_key_index = context.api_key_index;
3853
3854 let mut rollback_guard = TxDispatchGuard::new(
3855 self.dispatch.clone(),
3856 &self.credential,
3857 None,
3858 captured_nonce,
3859 );
3860
3861 let tx = ModifyOrderTxInfo {
3862 context,
3863 market_index: plan.market_index,
3864 index: plan.venue_index,
3865 base_amount: plan.base_amount,
3866 price: plan.price,
3867 trigger_price: plan.trigger_price,
3868 attributes: integrator_attributes(self.integrator_account_index),
3869 };
3870
3871 let signed = sign_tx(
3872 &tx,
3873 self.chain_id,
3874 &self.credential.private_key()?,
3875 fresh_k(),
3876 )?;
3877
3878 let tx_info_str = TxInfoJson::modify_order(&tx, &signed);
3879 let tx_info = serde_json::value::RawValue::from_string(tx_info_str)
3880 .context("failed to wrap signed Lighter modify tx_info JSON")?;
3881
3882 self.dispatch.set_pending_modify_nonce(
3883 &plan.client_order_id,
3884 plan.command_id,
3885 send_reservation.connection_epoch,
3886 captured_nonce,
3887 );
3888 rollback_guard.disarm();
3889
3890 Ok(PreparedModifyOrder {
3891 client_order_id: plan.client_order_id,
3892 strategy_id: plan.strategy_id,
3893 instrument_id: plan.instrument_id,
3894 venue_order_id: plan.venue_order_id,
3895 tx_info,
3896 nonce: captured_nonce,
3897 api_key_index: captured_api_key_index,
3898 tx_hash: signed.tx_hash_hex(),
3899 send_reservation,
3900 })
3901 }
3902
3903 fn sign_cancel_order(&self, plan: &CancelOrderPlan) -> anyhow::Result<PreparedCancelOrder> {
3904 let ReservedTxContext {
3905 context,
3906 send_reservation,
3907 } = self.build_tx_context()?;
3908
3909 let nonce = context.nonce;
3910 let api_key_index = context.api_key_index;
3911 let mut rollback_guard =
3912 TxDispatchGuard::new(self.dispatch.clone(), &self.credential, None, nonce);
3913
3914 let tx = CancelOrderTxInfo {
3915 context,
3916 market_index: plan.market_index,
3917 index: plan.venue_index,
3918 skip_nonce: 0,
3919 };
3920
3921 let signed = sign_tx(
3922 &tx,
3923 self.chain_id,
3924 &self.credential.private_key()?,
3925 fresh_k(),
3926 )?;
3927
3928 let tx_info =
3929 serde_json::value::RawValue::from_string(TxInfoJson::cancel_order(&tx, &signed))
3930 .context("failed to wrap signed Lighter cancel tx_info JSON")?;
3931
3932 rollback_guard.disarm();
3933
3934 Ok(PreparedCancelOrder {
3935 client_order_id: plan.client_order_id,
3936 strategy_id: plan.strategy_id,
3937 instrument_id: plan.instrument_id,
3938 venue_order_id: plan.venue_order_id,
3939 tx_info,
3940 nonce,
3941 api_key_index,
3942 tx_hash: signed.tx_hash_hex(),
3943 send_reservation,
3944 })
3945 }
3946
3947 async fn send_cancel_order(
3948 &self,
3949 mut prepared: PreparedCancelOrder,
3950 emit_cancel_rejected: bool,
3951 ) {
3952 let connection_epoch = prepared.send_reservation.connection_epoch;
3953 prepared.send_reservation.wait_for_turn().await;
3954 await_tx_quota(&self.tx_rate_limiter).await;
3955 self.enqueue_cancel_order(&prepared, emit_cancel_rejected);
3956 if let Err(e) = self
3957 .ws_client
3958 .send_tx_on_connection(
3959 LighterTxType::CancelOrder as u8,
3960 prepared.tx_info.clone(),
3961 connection_epoch,
3962 )
3963 .await
3964 {
3965 self.handle_cancel_send_failure(&prepared, emit_cancel_rejected, &e);
3966 }
3967
3968 prepared.send_reservation.release();
3969 }
3970
3971 async fn send_batch(&self, mut batch: Vec<PreparedBatchTx>) {
3972 let mut batch_ids = Vec::new();
3973
3974 while !batch.is_empty() {
3975 let count = batch
3978 .windows(2)
3979 .position(|pair| {
3980 pair[1].pending.nonce != pair[0].pending.nonce + 1
3981 || pair[1].send_reservation.connection_epoch
3982 != pair[0].send_reservation.connection_epoch
3983 })
3984 .map_or(batch.len(), |index| index + 1);
3985
3986 let mut group: Vec<_> = batch.drain(..count).collect();
3987 let connection_epoch = group[0].send_reservation.connection_epoch;
3988 let id = format!("{TX_BATCH_ID_PREFIX}{}", group[0].pending.tx_hash);
3989 group[0].send_reservation.wait_for_turn().await;
3990 await_tx_quota(&self.tx_rate_limiter).await;
3991
3992 let data = crate::websocket::messages::LighterWsSendTxBatch {
3993 id: id.clone(),
3994 tx_types: serde_json::to_string(
3995 &group
3996 .iter()
3997 .map(|prepared| prepared.tx_type)
3998 .collect::<Vec<_>>(),
3999 )
4000 .expect("transaction types serialize"),
4001 tx_infos: serde_json::to_string(
4002 &group
4003 .iter()
4004 .map(|prepared| prepared.tx_info.get())
4005 .collect::<Vec<_>>(),
4006 )
4007 .expect("transaction strings serialize"),
4008 };
4009
4010 for prepared in &mut group {
4011 prepared.pending.batch_id = Some(id.clone());
4012 prepared.pending.submitted_at = self.clock.get_time_ns();
4013 self.enqueue_batch_tx(prepared);
4014 }
4015
4016 if let Err(e) = self
4017 .ws_client
4018 .send_tx_batch_on_connection(data, connection_epoch)
4019 .await
4020 {
4021 for prepared in group.iter().rev() {
4022 self.handle_batch_send_failure(prepared, &e);
4023 }
4024 }
4025
4026 for prepared in &mut group {
4027 prepared.send_reservation.release();
4028 }
4029
4030 batch_ids.push(id);
4031 }
4032
4033 let wait = async {
4035 while self.dispatch.pending_sendtx.lock().iter().any(|pending| {
4036 pending
4037 .batch_id
4038 .as_ref()
4039 .is_some_and(|id| batch_ids.contains(id))
4040 }) {
4041 tokio::time::sleep(Duration::from_millis(10)).await;
4042 }
4043 };
4044
4045 if tokio::time::timeout(BATCH_ACK_TIMEOUT, wait).await.is_err() {
4046 log::warn!("Lighter batch acknowledgement timed out: ids={batch_ids:?}");
4047 }
4048 }
4049
4050 fn enqueue_batch_tx(&self, prepared: &PreparedBatchTx) {
4051 for (order, _) in prepared.pending.kind.creates() {
4052 self.emitter.emit_order_submitted(order);
4053 }
4054
4055 if let PendingSendTxKind::Cancel {
4056 client_order_id,
4057 emit_cancel_rejected,
4058 ..
4059 } = &prepared.pending.kind
4060 {
4061 self.dispatch.set_pending_cancel_nonce(
4062 client_order_id,
4063 prepared.send_reservation.connection_epoch,
4064 prepared.pending.nonce,
4065 *emit_cancel_rejected,
4066 );
4067 }
4068
4069 self.dispatch
4070 .enqueue_pending_sendtx(prepared.pending.clone());
4071 if matches!(prepared.pending.kind, PendingSendTxKind::Modify { .. }) {
4072 spawn_acked_order_probe(&prepared.pending, self.probe_context());
4073 }
4074 }
4075
4076 fn handle_batch_send_failure(&self, prepared: &PreparedBatchTx, error: &LighterWsError) {
4077 let action = match prepared.pending.kind {
4078 PendingSendTxKind::Create { .. } | PendingSendTxKind::CreateGroup { .. } => {
4079 "submit_order"
4080 }
4081 PendingSendTxKind::Modify { .. } => "modify_order",
4082 PendingSendTxKind::Cancel { .. } => "cancel_order",
4083 PendingSendTxKind::Other => {
4084 unreachable!("order batches contain only order transactions")
4085 }
4086 };
4087
4088 let failure = classify_lighter_ws_command_failure(action, error);
4089 let reason = command_failure_reason(&failure);
4090
4091 if matches!(failure, CommandFailure::Ambiguous(_)) {
4092 log::warn!("{reason}; retaining batch transaction for reconciliation");
4093 return;
4094 }
4095
4096 if let Some(pending) = self.dispatch.remove_pending_sendtx_by_hash(
4097 prepared.pending.connection_epoch,
4098 &prepared.pending.tx_hash,
4099 ) {
4100 reject_pending_sendtx(
4101 &self.dispatch,
4102 &self.emitter,
4103 Some(self.credential.account_index()),
4104 &pending,
4105 reason,
4106 self.clock.get_time_ns(),
4107 false,
4108 );
4109 }
4110 }
4111
4112 fn enqueue_cancel_order(&self, prepared: &PreparedCancelOrder, emit_cancel_rejected: bool) {
4113 self.dispatch.set_pending_cancel_nonce(
4114 &prepared.client_order_id,
4115 prepared.send_reservation.connection_epoch,
4116 prepared.nonce,
4117 emit_cancel_rejected,
4118 );
4119 self.dispatch.enqueue_pending_sendtx(PendingSendTx {
4120 batch_id: None,
4121 connection_epoch: prepared.send_reservation.connection_epoch,
4122 kind: if emit_cancel_rejected {
4123 PendingSendTxKind::Cancel {
4124 emit_cancel_rejected,
4125 strategy_id: prepared.strategy_id,
4126 instrument_id: prepared.instrument_id,
4127 client_order_id: prepared.client_order_id,
4128 venue_order_id: prepared.venue_order_id,
4129 }
4130 } else {
4131 PendingSendTxKind::Other
4132 },
4133 submitted_at: self.clock.get_time_ns(),
4134 nonce: prepared.nonce,
4135 api_key_index: prepared.api_key_index,
4136 tx_hash: prepared.tx_hash.clone(),
4137 });
4138 }
4139
4140 fn handle_cancel_send_failure(
4141 &self,
4142 prepared: &PreparedCancelOrder,
4143 emit_cancel_rejected: bool,
4144 error: &LighterWsError,
4145 ) {
4146 let failure = classify_lighter_ws_command_failure("cancel_order", error);
4147 let reason = command_failure_reason(&failure);
4148
4149 if matches!(failure, CommandFailure::Ambiguous(_)) {
4150 log::warn!(
4151 "Lighter cancel_order outcome unknown for {}: {reason}; retaining pending state for venue reconciliation",
4152 prepared.client_order_id
4153 );
4154 return;
4155 }
4156
4157 log::warn!(
4158 "{reason} for {}; diagnostic={error:?}",
4159 prepared.client_order_id
4160 );
4161
4162 self.dispatch.remove_pending_sendtx_by_nonce(
4163 prepared.send_reservation.connection_epoch,
4164 prepared.nonce,
4165 );
4166 let current = self.dispatch.clear_pending_cancel_if_nonce(
4167 &prepared.client_order_id,
4168 prepared.send_reservation.connection_epoch,
4169 prepared.nonce,
4170 );
4171 rollback_tx_dispatch(&self.dispatch, &self.credential, None, prepared.nonce);
4172
4173 if current && emit_cancel_rejected {
4174 self.emitter.emit_order_cancel_rejected_event(
4175 prepared.strategy_id,
4176 prepared.instrument_id,
4177 prepared.client_order_id,
4178 prepared.venue_order_id,
4179 reason,
4180 self.clock.get_time_ns(),
4181 );
4182 } else {
4183 log::warn!(
4184 "{reason} for {}; suppressing OrderCancelRejected because order is not PendingCancel",
4185 prepared.client_order_id
4186 );
4187 }
4188 }
4189}
4190
4191fn should_retry_batch_signing(error: &anyhow::Error) -> bool {
4192 error.is::<TxPreparationError>()
4193 || error.is::<tokio::sync::TryLockError>()
4194 || matches!(
4195 error.downcast_ref::<NonceError>(),
4196 Some(NonceError::SkipWindowExhausted { .. })
4197 )
4198}
4199
4200#[derive(Debug, thiserror::Error)]
4201enum TxPreparationError {
4202 #[error("Lighter nonce refresh is pending for connection epoch {0}")]
4203 NonceRefreshPending(u64),
4204 #[error("Lighter connection or nonce state changed during transaction preparation")]
4205 StateChanged,
4206}
4207
4208struct PreparedModifyOrder {
4209 client_order_id: ClientOrderId,
4210 strategy_id: StrategyId,
4211 instrument_id: InstrumentId,
4212 venue_order_id: Option<VenueOrderId>,
4213 tx_info: Box<serde_json::value::RawValue>,
4214 nonce: i64,
4215 api_key_index: u8,
4216 tx_hash: String,
4217 send_reservation: TxSendReservation,
4218}
4219
4220struct ModifyOrderPlan {
4221 command_id: UUID4,
4222 client_order_id: ClientOrderId,
4223 strategy_id: StrategyId,
4224 instrument_id: InstrumentId,
4225 venue_order_id: Option<VenueOrderId>,
4226 market_index: i64,
4227 venue_index: i64,
4228 base_amount: i64,
4229 price: u32,
4230 trigger_price: u32,
4231}
4232
4233enum BatchOrderPlan {
4234 Create(Box<CreateOrderPlan>),
4235 CreateGroup(CreateGroupPlan),
4236 Modify(ModifyOrderPlan),
4237 Cancel(CancelOrderPlan, bool),
4238}
4239
4240struct PreparedBatchTx {
4241 tx_type: u8,
4242 tx_info: Box<serde_json::value::RawValue>,
4243 pending: PendingSendTx,
4244 send_reservation: TxSendReservation,
4245}
4246
4247impl From<PreparedCreateOrder> for PreparedBatchTx {
4248 fn from(prepared: PreparedCreateOrder) -> Self {
4249 Self {
4250 tx_type: LighterTxType::CreateOrder as u8,
4251 tx_info: prepared.tx_info,
4252 pending: PendingSendTx {
4253 batch_id: None,
4254 connection_epoch: prepared.send_reservation.connection_epoch,
4255 kind: PendingSendTxKind::Create {
4256 order: Box::new(prepared.order),
4257 client_order_index: prepared.client_order_index,
4258 },
4259 submitted_at: UnixNanos::default(),
4260 nonce: prepared.nonce,
4261 api_key_index: prepared.api_key_index,
4262 tx_hash: prepared.tx_hash,
4263 },
4264 send_reservation: prepared.send_reservation,
4265 }
4266 }
4267}
4268
4269impl From<PreparedModifyOrder> for PreparedBatchTx {
4270 fn from(prepared: PreparedModifyOrder) -> Self {
4271 Self {
4272 tx_type: LighterTxType::ModifyOrder as u8,
4273 tx_info: prepared.tx_info,
4274 pending: PendingSendTx {
4275 batch_id: None,
4276 connection_epoch: prepared.send_reservation.connection_epoch,
4277 kind: PendingSendTxKind::Modify {
4278 client_order_id: prepared.client_order_id,
4279 strategy_id: prepared.strategy_id,
4280 instrument_id: prepared.instrument_id,
4281 venue_order_id: prepared.venue_order_id,
4282 },
4283 submitted_at: UnixNanos::default(),
4284 nonce: prepared.nonce,
4285 api_key_index: prepared.api_key_index,
4286 tx_hash: prepared.tx_hash,
4287 },
4288 send_reservation: prepared.send_reservation,
4289 }
4290 }
4291}
4292
4293impl From<(PreparedCancelOrder, bool)> for PreparedBatchTx {
4294 fn from((prepared, emit_cancel_rejected): (PreparedCancelOrder, bool)) -> Self {
4295 Self {
4296 tx_type: LighterTxType::CancelOrder as u8,
4297 tx_info: prepared.tx_info,
4298 pending: PendingSendTx {
4299 batch_id: None,
4300 connection_epoch: prepared.send_reservation.connection_epoch,
4301 kind: PendingSendTxKind::Cancel {
4302 emit_cancel_rejected,
4303 client_order_id: prepared.client_order_id,
4304 strategy_id: prepared.strategy_id,
4305 instrument_id: prepared.instrument_id,
4306 venue_order_id: prepared.venue_order_id,
4307 },
4308 submitted_at: UnixNanos::default(),
4309 nonce: prepared.nonce,
4310 api_key_index: prepared.api_key_index,
4311 tx_hash: prepared.tx_hash,
4312 },
4313 send_reservation: prepared.send_reservation,
4314 }
4315 }
4316}
4317
4318struct PreparedIntegratorApproval {
4319 tx_info: String,
4320 nonce: i64,
4321 api_key_index: u8,
4322 approval_expiry: i64,
4323 send_reservation: TxSendReservation,
4324}
4325
4326#[derive(Debug, PartialEq, Eq)]
4330enum TierCrossCheck {
4331 AboveTier { documented: u32 },
4332 RaiseHint { documented: u32 },
4333}
4334
4335fn tier_quota_report(
4340 tier: LighterAccountTier,
4341 rest_quota_per_min: Option<u32>,
4342 standard_rest: u32,
4343) -> (u32, Option<TierCrossCheck>) {
4344 let configured = rest_quota_per_min.filter(|&n| n > 0);
4345 let active_rest = configured.unwrap_or(standard_rest);
4346 let cross_check = match (tier.documented_rest_quota_per_min(), configured) {
4347 (Some(documented), Some(configured)) if configured > documented => {
4348 Some(TierCrossCheck::AboveTier { documented })
4349 }
4350 (Some(documented), None) if documented > standard_rest => {
4351 Some(TierCrossCheck::RaiseHint { documented })
4352 }
4353 _ => None,
4354 };
4355 (active_rest, cross_check)
4356}
4357
4358struct TxDispatchGuard {
4359 dispatch: WsDispatchState,
4360 account_index: i64,
4361 api_key_index: u8,
4362 client_order_index: Option<i64>,
4363 client_order_id: Option<ClientOrderId>,
4364 nonce: i64,
4365 armed: bool,
4366}
4367
4368impl TxDispatchGuard {
4369 fn new(
4370 dispatch: WsDispatchState,
4371 credential: &Credential,
4372 client_order_index: Option<i64>,
4373 nonce: i64,
4374 ) -> Self {
4375 Self {
4376 dispatch,
4377 account_index: credential.account_index(),
4378 api_key_index: credential.api_key_index(),
4379 client_order_index,
4380 client_order_id: None,
4381 nonce,
4382 armed: true,
4383 }
4384 }
4385
4386 fn with_order_identity(mut self, client_order_id: ClientOrderId) -> Self {
4387 self.client_order_id = Some(client_order_id);
4388 self
4389 }
4390
4391 fn disarm(&mut self) {
4392 self.armed = false;
4393 }
4394}
4395
4396impl Drop for TxDispatchGuard {
4397 fn drop(&mut self) {
4398 if self.armed {
4399 rollback_tx_dispatch_indices(
4400 &self.dispatch,
4401 self.account_index,
4402 self.api_key_index,
4403 self.client_order_index,
4404 self.client_order_id.as_ref(),
4405 self.nonce,
4406 );
4407 }
4408 }
4409}
4410
4411fn handle_send_tx_ack_for_connection(
4416 dispatch: &WsDispatchState,
4417 account_index: Option<i64>,
4418 connection_epoch: u64,
4419 code: i64,
4420 tx_hash: Option<&str>,
4421) -> Option<PendingSendTx> {
4422 let popped = match tx_hash {
4423 Some(hash) => {
4424 let matched = dispatch.remove_pending_sendtx_by_hash(connection_epoch, hash);
4425 if matched.is_none() {
4426 log::warn!("Lighter sendTx ack unmatched: tx_hash={hash} code={code}");
4427 }
4428 matched
4429 }
4430 None => dispatch.pop_pending_sendtx_if_only(connection_epoch),
4431 };
4432
4433 if let (Some(pending), Some(account_index)) = (&popped, account_index) {
4434 let _ =
4435 dispatch
4436 .nonce_manager
4437 .ack_success(account_index, pending.api_key_index, pending.nonce);
4438 }
4439
4440 log::debug!(
4441 "Lighter sendTx ack: code={code} tx_hash={tx_hash:?} popped_nonce={:?}",
4442 popped.as_ref().map(|p| p.nonce),
4443 );
4444
4445 popped
4446}
4447
4448#[expect(
4449 clippy::too_many_arguments,
4450 reason = "batch response shares the single-transaction rejection sink"
4451)]
4452fn handle_send_tx_batch_result(
4453 dispatch: &WsDispatchState,
4454 emitter: &ExecutionEventEmitter,
4455 account_index: Option<i64>,
4456 connection_epoch: u64,
4457 now: UnixNanos,
4458 id: &str,
4459 code: i64,
4460 message: &str,
4461 tx_hashes: &[String],
4462) -> (Vec<PendingSendTx>, bool) {
4463 let mut pending: Vec<_> = dispatch
4464 .pending_sendtx
4465 .lock()
4466 .iter()
4467 .filter(|pending| {
4468 pending.connection_epoch == connection_epoch && pending.batch_id.as_deref() == Some(id)
4469 })
4470 .cloned()
4471 .collect();
4472
4473 if pending.is_empty() {
4474 log::warn!("Ignoring unmatched Lighter batch response: id={id}");
4475 return (Vec::new(), false);
4476 }
4477
4478 if code == 200 {
4479 let acked = pending
4480 .iter()
4481 .filter(|pending| tx_hashes.contains(&pending.tx_hash))
4482 .filter_map(|pending| {
4483 handle_send_tx_ack_for_connection(
4484 dispatch,
4485 account_index,
4486 connection_epoch,
4487 code,
4488 Some(&pending.tx_hash),
4489 )
4490 })
4491 .collect();
4492
4493 if pending
4494 .iter()
4495 .any(|pending| !tx_hashes.contains(&pending.tx_hash))
4496 {
4497 log::warn!(
4498 "Incomplete Lighter batch acknowledgement: id={id}; retaining unmatched transactions for reconciliation"
4499 );
4500 }
4501
4502 return (acked, false);
4503 }
4504
4505 pending.sort_unstable_by_key(|pending| std::cmp::Reverse(pending.nonce));
4507 let mut resync = false;
4508 for pending in pending {
4509 resync |= handle_send_tx_rejection_for_connection(
4510 dispatch,
4511 emitter,
4512 account_index,
4513 connection_epoch,
4514 now,
4515 SendTxRejectionSource::Ack,
4516 Some(code),
4517 message,
4518 Some(&pending.tx_hash),
4519 );
4520 }
4521
4522 (Vec::new(), resync)
4523}
4524
4525fn spawn_acked_order_probe(pending: &PendingSendTx, context: AckedOrderProbeContext) {
4526 if let PendingSendTxKind::CreateGroup { orders } = &pending.kind {
4527 for (order, index) in orders {
4528 let mut leg = pending.clone();
4529
4530 leg.kind = PendingSendTxKind::Create {
4531 order: Box::new(order.clone()),
4532 client_order_index: *index,
4533 };
4534
4535 spawn_acked_order_probe(&leg, context.clone());
4536 }
4537
4538 return;
4539 }
4540
4541 let action = matches!(
4542 pending.kind,
4543 PendingSendTxKind::Modify { .. } | PendingSendTxKind::Cancel { .. }
4544 )
4545 .then(|| pending.clone());
4546 let probe = AckedOrderProbe::from_pending(pending);
4547 if action.is_none() && probe.is_none() {
4548 return;
4549 }
4550
4551 let pending_tasks = context.pending_tasks.clone();
4552
4553 let future = async move {
4554 tokio::select! {
4555 () = context.cancellation_token.cancelled() => return,
4556 () = tokio::time::sleep(ACKED_ORDER_LOOKUP_DELAY) => {}
4557 }
4558
4559 if let Some(pending) = action {
4560 let result = if let PendingSendTxKind::Cancel {
4561 client_order_id, ..
4562 } = pending.kind
4563 {
4564 let Some(token) = context.dispatch.pending_cancel_token(
4565 &client_order_id,
4566 pending.connection_epoch,
4567 pending.nonce,
4568 ) else {
4569 return;
4570 };
4571
4572 tokio::select! {
4573 biased;
4574 () = context.cancellation_token.cancelled() => return,
4575 () = token.cancelled() => return,
4576 result = probe_acked_action(&pending,&context) => result,
4577 }
4578 } else {
4579 probe_acked_action(&pending, &context).await
4580 };
4581
4582 if let Err(e) = result {
4583 log::warn!("Lighter acknowledged action probe failed: {e}");
4584 }
4585
4586 return;
4587 }
4588
4589 if let Some(probe) = probe
4590 && let Err(e) = probe_acked_create(&probe, &context).await
4591 {
4592 log::warn!("Lighter acknowledged order probe failed: {e:?}");
4593 }
4594 };
4595
4596 if let Err(e) = pending_tasks.spawn(future) {
4597 log::debug!("Skipping Lighter acknowledged-order probe during shutdown: {e}");
4598 }
4599}
4600
4601#[derive(Clone)]
4602struct AckedOrderProbeContext {
4603 http_client: LighterHttpClient,
4604 registry: Arc<MarketRegistry>,
4605 credential: Credential,
4606 dispatch: WsDispatchState,
4607 account_id: AccountId,
4608 clock: &'static AtomicTime,
4609 emitter: ExecutionEventEmitter,
4610 connection_epoch: Arc<AtomicU64>,
4611 cancellation_token: CancellationToken,
4612 pending_tasks: TaskSpawner,
4613}
4614
4615async fn probe_acked_action(
4616 pending: &PendingSendTx,
4617 context: &AckedOrderProbeContext,
4618) -> anyhow::Result<()> {
4619 let (client_order_id, _) = acked_action_identity(pending)?;
4620 for attempt in 1..=ACKED_ORDER_PROBE_ATTEMPTS {
4621 if !acked_action_is_pending(pending, context) {
4622 return Ok(());
4623 }
4624
4625 match context.http_client.get_tx(pending.tx_hash.clone()).await {
4626 Ok(tx) => match resolve_acked_action(pending, context, tx).await {
4627 Ok(true) => return Ok(()),
4628 Ok(false) => {}
4629 Err(e) => log::warn!("Lighter action confirmation unavailable: {e}"),
4630 },
4631 Err(LighterHttpError::Venue { code: 21500, .. }) => {}
4632 Err(e) => log::warn!("Lighter acknowledged action transaction lookup failed: {e}"),
4633 }
4634
4635 if attempt < ACKED_ORDER_PROBE_ATTEMPTS {
4636 tokio::select! {
4637 () = context.cancellation_token.cancelled() => return Ok(()),
4638 () = tokio::time::sleep(ACKED_ORDER_LOOKUP_DELAY) => {}
4639 }
4640 }
4641 }
4642
4643 log::warn!(
4644 "Lighter acknowledged action outcome unresolved for {client_order_id}; retaining pending action for reconciliation"
4645 );
4646 Ok(())
4647}
4648
4649fn acked_action_identity(
4650 pending: &PendingSendTx,
4651) -> anyhow::Result<(ClientOrderId, LighterTxType)> {
4652 match pending.kind {
4653 PendingSendTxKind::Cancel {
4654 client_order_id, ..
4655 } => Ok((client_order_id, LighterTxType::CancelOrder)),
4656 PendingSendTxKind::Modify {
4657 client_order_id, ..
4658 } => Ok((client_order_id, LighterTxType::ModifyOrder)),
4659 _ => anyhow::bail!("acknowledged action requires a cancel or modify transaction"),
4660 }
4661}
4662
4663fn acked_action_is_pending(pending: &PendingSendTx, context: &AckedOrderProbeContext) -> bool {
4664 let submission = match pending.kind {
4665 PendingSendTxKind::Cancel {
4666 client_order_id, ..
4667 } => context.dispatch.pending_cancel_nonce(&client_order_id),
4668 PendingSendTxKind::Modify {
4669 client_order_id, ..
4670 } => context.dispatch.pending_modify_nonce(&client_order_id),
4671 _ => return false,
4672 };
4673
4674 (matches!(pending.kind, PendingSendTxKind::Modify { .. })
4675 || context.connection_epoch.load(Ordering::Acquire) == pending.connection_epoch)
4676 && submission == Some((pending.connection_epoch, pending.nonce))
4677}
4678
4679async fn resolve_acked_action(
4680 pending: &PendingSendTx,
4681 context: &AckedOrderProbeContext,
4682 tx: LighterTx,
4683) -> anyhow::Result<bool> {
4684 let (_, tx_type) = acked_action_identity(pending)?;
4685 anyhow::ensure!(
4686 tx_hash_matches(&tx.hash, &pending.tx_hash)
4687 && tx.tx_type == tx_type as u8
4688 && tx.account_index == context.credential.account_index()
4689 && tx.api_key_index == pending.api_key_index
4690 && tx.nonce == pending.nonce,
4691 "Lighter transaction lookup did not match acknowledged action identity"
4692 );
4693
4694 let is_modify = tx_type == LighterTxType::ModifyOrder;
4695
4696 if !acked_action_is_pending(pending, context) {
4697 return Ok(true);
4698 }
4699
4700 let event = match parse_acked_tx_event(&tx.event_info) {
4701 Ok(event) => event,
4702 Err(e) if tx.status == LighterTxStatus::Failed => {
4703 log::warn!("Lighter failed transaction carried invalid event_info: {e}");
4704 AckedTxEvent::default()
4705 }
4706 Err(e) => return Err(e),
4707 };
4708
4709 if tx.status == LighterTxStatus::Failed || !event.app_error.is_empty() {
4710 let action = if is_modify { "modify" } else { "cancel" };
4711 let detail = acked_tx_error_detail(&event.app_error);
4712 let reason = format!("Lighter {action} transaction failed: {detail}");
4713 log::warn!("{reason} for nonce={}", pending.nonce);
4714
4715 match pending.kind {
4716 PendingSendTxKind::Modify { .. } => reject_modify_order(
4717 &context.dispatch,
4718 &context.emitter,
4719 pending,
4720 &reason,
4721 context.clock.get_time_ns(),
4722 ),
4723 PendingSendTxKind::Cancel { .. } => reject_cancel_order(
4724 &context.dispatch,
4725 &context.emitter,
4726 pending,
4727 &reason,
4728 context.clock.get_time_ns(),
4729 ),
4730 _ => unreachable!("validated acknowledged action"),
4731 }
4732
4733 return Ok(true);
4735 }
4736
4737 if tx.status == LighterTxStatus::Executed {
4738 return recover_acked_action_report(pending, context).await;
4739 }
4740
4741 Ok(false)
4742}
4743
4744async fn recover_acked_action_report(
4745 pending: &PendingSendTx,
4746 context: &AckedOrderProbeContext,
4747) -> anyhow::Result<bool> {
4748 let (instrument_id, client_order_id, venue_order_id) = match pending.kind {
4749 PendingSendTxKind::Cancel {
4750 instrument_id,
4751 client_order_id,
4752 venue_order_id,
4753 ..
4754 }
4755 | PendingSendTxKind::Modify {
4756 instrument_id,
4757 client_order_id,
4758 venue_order_id,
4759 ..
4760 } => (instrument_id, client_order_id, venue_order_id),
4761 _ => return Ok(true),
4762 };
4763
4764 let lookup = OrderReportLookup::new(
4765 &context.registry,
4766 &context.dispatch,
4767 Some(instrument_id),
4768 Some(&client_order_id),
4769 venue_order_id.as_ref(),
4770 )?;
4771
4772 anyhow::ensure!(
4773 matches!(pending.kind, PendingSendTxKind::Modify { .. })
4774 || !context.dispatch.has_pending_modify(&client_order_id),
4775 "Lighter cancellation waits for modification confirmation"
4776 );
4777 let order = collect_order_while_pending(
4778 &context.http_client,
4779 &context.registry,
4780 &context.credential,
4781 &lookup,
4782 || acked_action_is_pending(pending, context),
4783 )
4784 .await?;
4785
4786 if !acked_action_is_pending(pending, context) {
4787 return Ok(true);
4788 }
4789
4790 let Some(order) = order else {
4791 return Ok(false);
4792 };
4793
4794 if matches!(pending.kind, PendingSendTxKind::Modify { .. }) {
4795 confirm_modify_order(pending, context, &order)?;
4796 return Ok(true);
4797 }
4798
4799 recover_report_fills(&order, context).await?;
4800
4801 let events = context.dispatch.order_events.lock();
4802 if context
4803 .dispatch
4804 .terminals_pending
4805 .lock()
4806 .contains_key(&client_order_id)
4807 || events.get(&client_order_id).is_some_and(|deferred| {
4808 deferred
4809 .order
4810 .as_ref()
4811 .is_some_and(|order| order.status.is_terminal())
4812 })
4813 || context
4814 .dispatch
4815 .order_identity(&client_order_id)
4816 .is_some_and(|identity| position_fills_pending(&identity, &events, &context.dispatch))
4817 {
4818 return Ok(false);
4819 }
4820
4821 let report = lookup.parse(
4822 &order,
4823 &context.registry,
4824 context.account_id,
4825 context.clock.get_time_ns(),
4826 &context.dispatch,
4827 )?;
4828 let report = lookup.translate(&context.dispatch, report);
4829 if context
4830 .dispatch
4831 .order_identity(&client_order_id)
4832 .is_some_and(|identity| {
4833 !identity.report_fills_covered(&report, &[]) || identity.has_publication()
4834 })
4835 {
4836 return Ok(false);
4837 }
4838
4839 let resolved = report.order_status.is_closed();
4840
4841 let current = if resolved {
4842 context.dispatch.clear_pending_cancel_if_nonce(
4843 &client_order_id,
4844 pending.connection_epoch,
4845 pending.nonce,
4846 )
4847 } else {
4848 context.dispatch.pending_cancel_nonce(&client_order_id)
4849 == Some((pending.connection_epoch, pending.nonce))
4850 };
4851
4852 if !current {
4853 return Ok(true);
4854 }
4855
4856 let report = if report.order_status.is_closed() {
4857 report
4858 } else {
4859 context.dispatch.preserve_pending_order_status(report)
4860 };
4861
4862 context.emitter.send_order_status_report(report);
4863 Ok(resolved)
4864}
4865
4866fn confirm_modify_order(
4867 pending: &PendingSendTx,
4868 context: &AckedOrderProbeContext,
4869 order: &crate::http::models::LighterOrder,
4870) -> anyhow::Result<()> {
4871 let (cloid, _) = acked_action_identity(pending)?;
4872 let mut events = context.dispatch.order_events.lock();
4873 if !acked_action_is_pending(pending, context) {
4874 return Ok(());
4875 }
4876
4877 let identity = context
4878 .dispatch
4879 .order_identity(&cloid)
4880 .context("missing modified order identity")?;
4881 let instrument = LIGHTER_INSTRUMENT_CACHE
4882 .get(&identity.instrument_id)
4883 .context("missing modified order instrument")?;
4884 let shape = lighter_order_shape(order, &instrument, identity.order_type)?;
4885 let mut requested_shape = shape.clone();
4886 requested_shape.quantity =
4887 quantity_from_decimal(order.initial_base_amount, instrument.size_precision())?;
4888 anyhow::ensure!(
4889 context.dispatch.pending_modify_shape(&cloid).as_ref() == Some(&requested_shape),
4890 "Lighter executed modification shape is not available yet"
4891 );
4892 identity.record_reduction(order.initial_base_amount - shape.quantity.as_decimal());
4893 let updated = parse_lighter_order_updated(
4894 order,
4895 &instrument,
4896 &identity,
4897 cloid,
4898 context.account_id,
4899 context.emitter.trader_id(),
4900 context.clock.get_time_ns(),
4901 )?;
4902 context.dispatch.store_snapshot(cloid, shape);
4903 identity.publish_update(updated.event_id);
4904 context
4905 .emitter
4906 .send_order_event(OrderEventAny::Updated(updated));
4907 context
4908 .dispatch
4909 .clear_pending_modify_if_nonce(&cloid, pending.connection_epoch, pending.nonce);
4910 let mut deferred = events.remove(&cloid).unwrap_or_default();
4911 if deferred.order.is_none() {
4912 deferred.store_order(order);
4913 }
4914
4915 flush_modify_events(
4916 cloid,
4917 deferred,
4918 &context.dispatch,
4919 &context.emitter,
4920 &mut events,
4921 );
4922 drop(events);
4923 spawn_child_recoveries(context);
4924 Ok(())
4925}
4926
4927fn flush_modify_events(
4928 cloid: ClientOrderId,
4929 deferred: DeferredOrderEvents,
4930 dispatch: &WsDispatchState,
4931 emitter: &ExecutionEventEmitter,
4932 events: &mut AHashMap<ClientOrderId, DeferredOrderEvents>,
4933) {
4934 if dispatch
4935 .order_identity(&cloid)
4936 .is_some_and(|identity| identity.child_fill_baseline().is_some())
4937 && let Some(order) = deferred
4938 .order
4939 .as_ref()
4940 .filter(|order| is_order_triggered(order))
4941 {
4942 dispatch_deferred_order(cloid, order, dispatch, emitter);
4943 }
4944
4945 events.insert(cloid, deferred);
4946 publish_child_recovery(cloid, events, dispatch, emitter);
4947}
4948
4949fn flush_order_terminal(
4950 cloid: ClientOrderId,
4951 deferred: DeferredOrderEvents,
4952 dispatch: &WsDispatchState,
4953 emitter: &ExecutionEventEmitter,
4954 events: &AHashMap<ClientOrderId, DeferredOrderEvents>,
4955) {
4956 for fill in deferred.fills {
4957 if dispatch.mark_trade_seen(fill.trade_id) {
4958 dispatch_lighter_fill(fill, emitter, dispatch, events);
4959 }
4960 }
4961
4962 let Some(order) = deferred.order else {
4963 return;
4964 };
4965
4966 dispatch_deferred_order(cloid, &order, dispatch, emitter);
4967}
4968
4969fn dispatch_deferred_order(
4970 cloid: ClientOrderId,
4971 order: &crate::http::models::LighterOrder,
4972 dispatch: &WsDispatchState,
4973 emitter: &ExecutionEventEmitter,
4974) {
4975 let Some(identity) = dispatch
4976 .order_identities
4977 .get(&cloid)
4978 .map(|entry| entry.value().clone())
4979 else {
4980 return;
4981 };
4982
4983 let Some(instrument) = LIGHTER_INSTRUMENT_CACHE.get(&identity.instrument_id) else {
4984 return;
4985 };
4986
4987 let ts_init = get_atomic_clock_realtime().get_time_ns();
4988
4989 if order.status == crate::common::enums::LighterOrderStatus::Filled {
4990 dispatch_owned_order(
4991 order,
4992 &instrument,
4993 &identity,
4994 cloid,
4995 VenueOrderId::new(&order.order_id),
4996 dispatch,
4997 emitter,
4998 emitter.account_id(),
4999 emitter.trader_id(),
5000 ts_init,
5001 );
5002 return;
5003 }
5004
5005 let context = OpenFrameContext {
5006 accepted_already_emitted: dispatch.accepted_was_emitted(&cloid),
5007 triggered_already_emitted: dispatch.triggered_was_emitted(&cloid),
5008 shape_changed: false,
5009 };
5010
5011 match parse_lighter_order_event(
5012 order,
5013 &instrument,
5014 &identity,
5015 cloid,
5016 emitter.account_id(),
5017 emitter.trader_id(),
5018 context,
5019 ts_init,
5020 ) {
5021 Ok(Some(event)) => {
5022 let Some(event) = defer_terminal(order, event, &identity, cloid, dispatch) else {
5023 return;
5024 };
5025
5026 dispatch_tracked_order_event(
5027 event,
5028 cloid,
5029 VenueOrderId::new(&order.order_id),
5030 &identity,
5031 emitter.account_id(),
5032 emitter.trader_id(),
5033 emitter,
5034 dispatch,
5035 ts_init,
5036 );
5037 }
5038 Ok(None) => {}
5039 Err(e) => log::error!("Failed to parse deferred Lighter order for {cloid}: {e}"),
5040 }
5041}
5042
5043async fn probe_acked_create(
5044 probe: &AckedOrderProbe,
5045 context: &AckedOrderProbeContext,
5046) -> anyhow::Result<()> {
5047 let AckedOrderProbe {
5048 order,
5049 client_order_index,
5050 connection_epoch,
5051 nonce,
5052 ..
5053 } = probe;
5054
5055 let client_order_id = order.client_order_id();
5056 let mut transaction_executed = false;
5057
5058 for attempt in 1..=ACKED_ORDER_PROBE_ATTEMPTS {
5059 if context.connection_epoch.load(Ordering::Acquire) != *connection_epoch {
5060 log::warn!(
5061 "Lighter acknowledged create outcome unresolved after reconnect for {client_order_id}; retaining identity for reconciliation",
5062 );
5063 return Ok(());
5064 }
5065
5066 if !context.dispatch.create_submission_is_pending(
5067 &client_order_id,
5068 *client_order_index,
5069 *nonce,
5070 ) {
5071 return Ok(());
5072 }
5073
5074 if probe_acked_create_report(probe, context, attempt).await {
5075 return Ok(());
5076 }
5077
5078 match probe_acked_create_tx(probe, context, attempt).await? {
5079 ControlFlow::Break(()) => return Ok(()),
5080 ControlFlow::Continue(executed) => transaction_executed |= executed,
5081 }
5082
5083 if attempt < ACKED_ORDER_PROBE_ATTEMPTS {
5084 tokio::select! {
5085 () = context.cancellation_token.cancelled() => return Ok(()),
5086 () = tokio::time::sleep(ACKED_ORDER_LOOKUP_DELAY) => {}
5087 }
5088 }
5089 }
5090
5091 if transaction_executed {
5092 let _ = context.dispatch.confirm_create_submission(
5093 &client_order_id,
5094 *client_order_index,
5095 *nonce,
5096 );
5097 log::warn!(
5098 "Lighter acknowledged create transaction executed without a queryable order for {client_order_id}; retaining identity for reconciliation",
5099 );
5100 } else {
5101 log::warn!(
5102 "Lighter acknowledged create outcome unresolved after {ACKED_ORDER_PROBE_ATTEMPTS} transaction lookups for {client_order_id}; retaining identity for reconciliation",
5103 );
5104 }
5105 Ok(())
5106}
5107
5108async fn probe_acked_create_report(
5109 probe: &AckedOrderProbe,
5110 context: &AckedOrderProbeContext,
5111 attempt: usize,
5112) -> bool {
5113 let AckedOrderProbe {
5114 order,
5115 client_order_index,
5116 nonce,
5117 ..
5118 } = probe;
5119
5120 let client_order_id = order.client_order_id();
5121
5122 let report = match lookup_create_order_status_report(
5123 &context.http_client,
5124 &context.registry,
5125 &context.credential,
5126 context.account_id,
5127 order.instrument_id(),
5128 client_order_id,
5129 *client_order_index,
5130 *nonce,
5131 &context.dispatch,
5132 context.clock,
5133 )
5134 .await
5135 {
5136 Ok(report) => report,
5137 Err(e) => {
5138 log::warn!(
5139 "Lighter acknowledged create order lookup failed for {client_order_id} on attempt {attempt}: {e:?}",
5140 );
5141 None
5142 }
5143 };
5144
5145 if let Some(report) = report {
5146 let _events = context.dispatch.order_events.lock();
5147 if context
5148 .dispatch
5149 .order_identity(&client_order_id)
5150 .is_some_and(|identity| {
5151 !identity.report_fills_covered(&report, &[]) || identity.has_publication()
5152 })
5153 {
5154 return false;
5155 }
5156
5157 if context.dispatch.observe_create_submission(
5158 &client_order_id,
5159 *client_order_index,
5160 *nonce,
5161 report.venue_order_id,
5162 ) {
5163 context.dispatch.seed_accepted_from_report(&report);
5164 context.emitter.send_order_status_report(report);
5165 }
5166
5167 return true;
5168 }
5169
5170 false
5171}
5172
5173async fn probe_acked_create_tx(
5174 probe: &AckedOrderProbe,
5175 context: &AckedOrderProbeContext,
5176 attempt: usize,
5177) -> anyhow::Result<ControlFlow<(), bool>> {
5178 let AckedOrderProbe {
5179 order,
5180 client_order_index,
5181 connection_epoch,
5182 nonce,
5183 api_key_index,
5184 tx_hash,
5185 } = probe;
5186
5187 let client_order_id = order.client_order_id();
5188
5189 let tx = match context.http_client.get_tx(tx_hash.clone()).await {
5190 Ok(tx) => tx,
5191 Err(LighterHttpError::Venue { code: 21500, .. }) => {
5192 return Ok(ControlFlow::Continue(false));
5193 }
5194 Err(e) => {
5195 log::warn!(
5196 "Lighter acknowledged create transaction lookup failed for {client_order_id} on attempt {attempt}: {e}",
5197 );
5198 return Ok(ControlFlow::Continue(false));
5199 }
5200 };
5201
5202 validate_acked_create_tx(
5203 &tx,
5204 context.credential.account_index(),
5205 *api_key_index,
5206 *client_order_index,
5207 *nonce,
5208 tx_hash,
5209 )?;
5210
5211 let event = parse_acked_tx_event(&tx.event_info).unwrap_or_else(|e| {
5212 log::warn!(
5213 "Lighter create transaction carried invalid event_info for {client_order_id}: {e}",
5214 );
5215 AckedTxEvent::default()
5216 });
5217
5218 if tx.status == LighterTxStatus::Failed || !event.app_error.is_empty() {
5219 let detail = acked_tx_error_detail(&event.app_error);
5220 let reason = format!(
5221 "Lighter sequencer rejected acknowledged create transaction {tx_hash}: {detail}",
5222 );
5223 reject_create_order(
5224 &context.dispatch,
5225 &context.emitter,
5226 order,
5227 *client_order_index,
5228 *nonce,
5229 &reason,
5230 context.clock.get_time_ns(),
5231 lighter_reason_indicates_post_only_rejection(detail),
5232 Some((&context.connection_epoch, *connection_epoch)),
5233 );
5234 return Ok(ControlFlow::Break(()));
5235 }
5236
5237 Ok(ControlFlow::Continue(
5238 tx.status == LighterTxStatus::Executed,
5239 ))
5240}
5241
5242#[derive(Deserialize)]
5243struct AckedCreateTxInfo {
5244 #[serde(rename = "ClientOrderIndex")]
5245 client_order_index: i64,
5246}
5247
5248#[derive(Deserialize)]
5249struct AckedCreateGroupTxInfo {
5250 #[serde(rename = "Orders")]
5251 orders: Vec<AckedCreateTxInfo>,
5252}
5253
5254#[derive(Default, Deserialize)]
5255struct AckedTxEvent {
5256 #[serde(default, rename = "ae")]
5257 app_error: String,
5258}
5259
5260fn validate_acked_create_tx(
5261 tx: &crate::http::models::LighterTx,
5262 account_index: i64,
5263 api_key_index: u8,
5264 client_order_index: i64,
5265 nonce: i64,
5266 tx_hash: &str,
5267) -> anyhow::Result<()> {
5268 anyhow::ensure!(
5269 tx_hash_matches(&tx.hash, tx_hash)
5270 && matches!(tx.tx_type, value if value == LighterTxType::CreateOrder as u8 || value == LighterTxType::CreateGroupedOrders as u8)
5271 && tx.account_index == account_index
5272 && tx.api_key_index == api_key_index
5273 && tx.nonce == nonce,
5274 "Lighter transaction lookup did not match acknowledged create identity",
5275 );
5276
5277 if tx.tx_type == LighterTxType::CreateGroupedOrders as u8 {
5278 let info: AckedCreateGroupTxInfo = serde_json::from_str(&tx.info)
5279 .context("failed to parse Lighter grouped transaction info")?;
5280 anyhow::ensure!(
5281 info.orders
5282 .iter()
5283 .any(|order| order.client_order_index == client_order_index),
5284 "Lighter transaction lookup did not include acknowledged client_order_index {client_order_index}"
5285 );
5286 return Ok(());
5287 }
5288
5289 let info: AckedCreateTxInfo = serde_json::from_str(&tx.info)
5290 .context("failed to parse Lighter create transaction info")?;
5291 anyhow::ensure!(
5292 info.client_order_index == client_order_index,
5293 "Lighter transaction lookup returned client_order_index {} for acknowledged create {client_order_index}",
5294 info.client_order_index,
5295 );
5296 Ok(())
5297}
5298
5299fn tx_hash_matches(left: &str, right: &str) -> bool {
5300 let left = left
5301 .strip_prefix("0x")
5302 .or_else(|| left.strip_prefix("0X"))
5303 .unwrap_or(left);
5304 let right = right
5305 .strip_prefix("0x")
5306 .or_else(|| right.strip_prefix("0X"))
5307 .unwrap_or(right);
5308 left.eq_ignore_ascii_case(right)
5309}
5310
5311fn parse_acked_tx_event(event_info: &str) -> anyhow::Result<AckedTxEvent> {
5312 if event_info.trim().is_empty() {
5313 return Ok(AckedTxEvent::default());
5314 }
5315
5316 serde_json::from_str(event_info).context("failed to parse Lighter transaction event")
5317}
5318
5319fn acked_tx_error_detail(app_error: &str) -> &str {
5320 if app_error.is_empty() {
5321 "transaction failed without an application error"
5322 } else {
5323 app_error
5324 }
5325}
5326
5327#[derive(Debug, Clone)]
5328struct AckedOrderProbe {
5329 order: Box<OrderAny>,
5330 client_order_index: i64,
5331 connection_epoch: u64,
5332 nonce: i64,
5333 api_key_index: u8,
5334 tx_hash: String,
5335}
5336
5337impl AckedOrderProbe {
5338 fn from_pending(pending: &PendingSendTx) -> Option<Self> {
5339 let PendingSendTxKind::Create {
5340 order,
5341 client_order_index,
5342 } = &pending.kind
5343 else {
5344 return None;
5345 };
5346
5347 Some(Self {
5348 order: order.clone(),
5349 client_order_index: *client_order_index,
5350 connection_epoch: pending.connection_epoch,
5351 nonce: pending.nonce,
5352 api_key_index: pending.api_key_index,
5353 tx_hash: pending.tx_hash.clone(),
5354 })
5355 }
5356}
5357
5358#[expect(
5367 clippy::too_many_arguments,
5368 reason = "shared terminal create transition keeps cleanup and event attribution together"
5369)]
5370fn reject_create_order(
5371 dispatch: &WsDispatchState,
5372 emitter: &ExecutionEventEmitter,
5373 order: &OrderAny,
5374 client_order_index: i64,
5375 nonce: i64,
5376 reason: &str,
5377 now: UnixNanos,
5378 due_post_only: bool,
5379 connection_epoch: Option<(&AtomicU64, u64)>,
5380) -> bool {
5381 let cloid = order.client_order_id();
5382 if !dispatch.reject_create_submission(&cloid, client_order_index, nonce, connection_epoch) {
5383 log::warn!(
5384 "Ignored stale Lighter create rejection for cloid={cloid} client_order_index={client_order_index} nonce={nonce}",
5385 );
5386 return false;
5387 }
5388 emitter.emit_order_rejected(order, reason, now, due_post_only);
5389 true
5390}
5391
5392#[expect(
5393 clippy::too_many_arguments,
5394 reason = "consumer-loop sink that flattens one SendTxRejected message without a wrapper struct"
5395)]
5396fn handle_send_tx_rejection_for_connection(
5397 dispatch: &WsDispatchState,
5398 emitter: &ExecutionEventEmitter,
5399 account_index: Option<i64>,
5400 connection_epoch: u64,
5401 now: UnixNanos,
5402 source: SendTxRejectionSource,
5403 code: Option<i64>,
5404 message: &str,
5405 tx_hash: Option<&str>,
5406) -> bool {
5407 let needs_nonce_resync = code == Some(LIGHTER_ERROR_CODE_INVALID_NONCE);
5408 let failure = venue_rejection_failure(code, message);
5409 let reason = command_failure_reason(&failure);
5410
5411 let pending = match tx_hash {
5412 Some(hash) => dispatch.remove_pending_sendtx_by_hash(connection_epoch, hash),
5413 None => match source {
5414 SendTxRejectionSource::Ack => dispatch.pop_pending_sendtx_if_only(connection_epoch),
5415 SendTxRejectionSource::BareError => dispatch.pop_pending_sendtx_if_only_within(
5416 connection_epoch,
5417 now,
5418 SENDTX_BARE_ERROR_WINDOW_MS,
5419 ),
5420 },
5421 };
5422 let Some(pending) = pending else {
5423 log::warn!(
5424 "Lighter sendTx rejection unattributed (source={source:?} code={code:?}): {message:?}",
5425 );
5426 return needs_nonce_resync;
5427 };
5428
5429 reject_pending_sendtx(
5430 dispatch,
5431 emitter,
5432 account_index,
5433 &pending,
5434 reason,
5435 now,
5436 lighter_reason_indicates_post_only_rejection(message),
5437 );
5438 needs_nonce_resync
5439}
5440
5441fn reject_pending_sendtx(
5442 dispatch: &WsDispatchState,
5443 emitter: &ExecutionEventEmitter,
5444 account_index: Option<i64>,
5445 pending: &PendingSendTx,
5446 reason: &str,
5447 now: UnixNanos,
5448 due_post_only: bool,
5449) {
5450 log::warn!(
5451 "{reason}, nonce={} api_key_index={}",
5452 pending.nonce,
5453 pending.api_key_index
5454 );
5455
5456 if let Some(account_index) = account_index {
5457 let _ = dispatch.nonce_manager.ack_failure_if_latest(
5458 account_index,
5459 pending.api_key_index,
5460 pending.nonce,
5461 );
5462 }
5463
5464 for (order, client_order_index) in pending.kind.creates() {
5465 reject_create_order(
5466 dispatch,
5467 emitter,
5468 order,
5469 client_order_index,
5470 pending.nonce,
5471 reason,
5472 now,
5473 due_post_only,
5474 None,
5475 );
5476 }
5477
5478 match &pending.kind {
5479 PendingSendTxKind::Create { .. } | PendingSendTxKind::CreateGroup { .. } => {}
5480 PendingSendTxKind::Cancel { .. } => {
5481 reject_cancel_order(dispatch, emitter, pending, reason, now);
5482 }
5483 PendingSendTxKind::Modify { .. } => {
5484 reject_modify_order(dispatch, emitter, pending, reason, now);
5485 }
5486 PendingSendTxKind::Other => {}
5487 }
5488}
5489
5490fn reject_cancel_order(
5491 dispatch: &WsDispatchState,
5492 emitter: &ExecutionEventEmitter,
5493 pending: &PendingSendTx,
5494 reason: &str,
5495 now: UnixNanos,
5496) {
5497 let PendingSendTxKind::Cancel {
5498 emit_cancel_rejected,
5499 strategy_id,
5500 instrument_id,
5501 client_order_id,
5502 venue_order_id,
5503 } = pending.kind
5504 else {
5505 return;
5506 };
5507
5508 if dispatch.clear_pending_cancel_if_nonce(
5509 &client_order_id,
5510 pending.connection_epoch,
5511 pending.nonce,
5512 ) && emit_cancel_rejected
5513 {
5514 emitter.emit_order_cancel_rejected_event(
5515 strategy_id,
5516 instrument_id,
5517 client_order_id,
5518 venue_order_id,
5519 reason,
5520 now,
5521 );
5522 }
5523}
5524
5525fn reject_modify_order(
5526 dispatch: &WsDispatchState,
5527 emitter: &ExecutionEventEmitter,
5528 pending: &PendingSendTx,
5529 reason: &str,
5530 now: UnixNanos,
5531) {
5532 let PendingSendTxKind::Modify {
5533 strategy_id,
5534 instrument_id,
5535 client_order_id,
5536 venue_order_id,
5537 } = pending.kind
5538 else {
5539 return;
5540 };
5541
5542 let mut events = dispatch.order_events.lock();
5543 if dispatch.clear_pending_modify_if_nonce(
5544 &client_order_id,
5545 pending.connection_epoch,
5546 pending.nonce,
5547 ) {
5548 emitter.emit_order_modify_rejected_event(
5549 strategy_id,
5550 instrument_id,
5551 client_order_id,
5552 venue_order_id,
5553 reason,
5554 now,
5555 );
5556
5557 if let Some(deferred) = events.remove(&client_order_id) {
5558 flush_modify_events(client_order_id, deferred, dispatch, emitter, &mut events);
5559 }
5560 }
5561}
5562
5563fn lighter_reason_indicates_post_only_rejection(reason: &str) -> bool {
5564 let normalized: String = reason
5565 .chars()
5566 .filter_map(|ch| {
5567 if ch == '-' || ch == '_' || ch.is_whitespace() {
5568 None
5569 } else {
5570 Some(ch.to_ascii_lowercase())
5571 }
5572 })
5573 .collect();
5574
5575 normalized.contains("postonly") || normalized.contains("postwouldexecute")
5576}
5577
5578fn lighter_http_error_is_definite_api_rejection(error: &LighterHttpError) -> bool {
5579 match error {
5580 LighterHttpError::RateLimit { .. } | LighterHttpError::Venue { .. } => true,
5581 LighterHttpError::Http { status, .. } => *status < 500,
5582 LighterHttpError::Network(_)
5583 | LighterHttpError::HistoryIncomplete { .. }
5584 | LighterHttpError::Parse(_) => false,
5585 }
5586}
5587
5588fn rollback_tx_dispatch(
5589 dispatch: &WsDispatchState,
5590 credential: &Credential,
5591 client_order_index: Option<i64>,
5592 nonce: i64,
5593) {
5594 rollback_tx_dispatch_indices(
5595 dispatch,
5596 credential.account_index(),
5597 credential.api_key_index(),
5598 client_order_index,
5599 None,
5600 nonce,
5601 );
5602}
5603
5604fn rollback_tx_dispatch_create(
5605 dispatch: &WsDispatchState,
5606 credential: &Credential,
5607 client_order_index: Option<i64>,
5608 client_order_id: &ClientOrderId,
5609 nonce: i64,
5610) {
5611 rollback_tx_dispatch_indices(
5612 dispatch,
5613 credential.account_index(),
5614 credential.api_key_index(),
5615 client_order_index,
5616 Some(client_order_id),
5617 nonce,
5618 );
5619}
5620
5621fn rollback_tx_dispatch_indices(
5625 dispatch: &WsDispatchState,
5626 account_index: i64,
5627 api_key_index: u8,
5628 client_order_index: Option<i64>,
5629 client_order_id: Option<&ClientOrderId>,
5630 nonce: i64,
5631) {
5632 let _ = dispatch
5633 .nonce_manager
5634 .ack_failure_if_latest(account_index, api_key_index, nonce);
5635
5636 if let Some(client_order_index) = client_order_index {
5637 dispatch.forget_cloid(client_order_index);
5638 }
5639
5640 if let Some(cloid) = client_order_id {
5641 dispatch.forget_order_identity(cloid);
5642 }
5643}
5644
5645fn integrator_attributes(integrator_account_index: Option<u64>) -> L2TxAttributes {
5646 integrator_account_index.map_or_else(L2TxAttributes::default, |integrator_account_index| {
5647 L2TxAttributes {
5648 integrator_account_index,
5649 ..Default::default()
5650 }
5651 })
5652}
5653
5654fn command_failure_reason(failure: &CommandFailure) -> &str {
5655 match failure {
5656 CommandFailure::NotSent(reason)
5657 | CommandFailure::Ambiguous(reason)
5658 | CommandFailure::VenueRejected(reason) => reason,
5659 }
5660}
5661
5662fn warn_pending_sendtx_unknown(pending_sendtx: &[PendingSendTx], context: &str) {
5663 if pending_sendtx.is_empty() {
5664 return;
5665 }
5666
5667 log::warn!(
5668 "Discarded {} pending sendTx entries during {context}; order state recovers via \
5669 reconciliation",
5670 pending_sendtx.len(),
5671 );
5672
5673 for pending in pending_sendtx {
5674 for (order, _) in pending.kind.creates() {
5675 log::warn!(
5676 "Lighter sendTx outcome unknown during {context} for {}",
5677 order.client_order_id(),
5678 );
5679 }
5680 }
5681}
5682
5683fn classify_lighter_ws_command_failure(action: &str, error: &LighterWsError) -> CommandFailure {
5684 let fallback = format!("Lighter {action} dispatch failed");
5685 let reason = sanitize_strategy_reason(&format!("{fallback}: {error}"))
5686 .unwrap_or_else(|| fallback.clone());
5687
5688 match error {
5689 LighterWsError::SendTxOutcomeUnknown(_)
5690 | LighterWsError::Network(_)
5691 | LighterWsError::Parse(_)
5692 | LighterWsError::Transport(SendError::WriteTimeout | SendError::BrokenPipe(_)) => {
5693 CommandFailure::ambiguous(reason)
5694 }
5695 LighterWsError::Transport(
5696 SendError::InvalidInput(_)
5697 | SendError::BufferFull
5698 | SendError::Closed
5699 | SendError::Timeout
5700 | SendError::ConnectionChanged,
5701 )
5702 | LighterWsError::Authentication(_)
5703 | LighterWsError::Client(_) => CommandFailure::not_sent(reason),
5704 }
5705}
5706
5707fn venue_rejection_failure(code: Option<i64>, message: &str) -> CommandFailure {
5708 let clean_message = sanitize_strategy_reason(message);
5709 let reason = match (code, clean_message) {
5710 (Some(code), Some(message)) => format!("LIGHTER_{code}: {message}"),
5711 (Some(code), None) => format!("LIGHTER_{code}"),
5712 (None, Some(message)) => message,
5713 (None, None) => "Lighter venue rejected sendTx".to_string(),
5714 };
5715 let reason = sanitize_strategy_reason(&reason)
5716 .unwrap_or_else(|| "Lighter venue rejected sendTx".to_string());
5717 CommandFailure::venue_rejected(reason)
5718}
5719
5720fn sanitize_strategy_reason(input: &str) -> Option<String> {
5721 let mut output = String::new();
5722 let mut inside_markup = false;
5723 let mut pending_space = false;
5724 let mut char_count = 0;
5725 let mut chars = input.chars().peekable();
5726
5727 while let Some(ch) = chars.next() {
5728 if inside_markup {
5729 if ch == '>' {
5730 inside_markup = false;
5731 }
5732 continue;
5733 }
5734
5735 match ch {
5736 '<' if chars.peek().is_some_and(|next| {
5737 next.is_ascii_alphabetic() || matches!(next, '/' | '!' | '?')
5738 }) =>
5739 {
5740 inside_markup = true;
5741 pending_space = !output.is_empty();
5742 }
5743 _ if ch.is_control() || is_unicode_format_control(ch) || ch.is_whitespace() => {
5744 pending_space = !output.is_empty();
5745 }
5746 _ => {
5747 if pending_space && char_count < STRATEGY_REASON_MAX_CHARS {
5748 output.push(' ');
5749 char_count += 1;
5750 }
5751 pending_space = false;
5752
5753 if char_count == STRATEGY_REASON_MAX_CHARS {
5754 break;
5755 }
5756 output.push(ch);
5757 char_count += 1;
5758 }
5759 }
5760 }
5761
5762 let output = output.trim().to_string();
5763 (!output.is_empty()).then_some(output)
5764}
5765
5766fn is_unicode_format_control(ch: char) -> bool {
5767 matches!(
5768 ch,
5769 '\u{00ad}'
5770 | '\u{061c}'
5771 | '\u{06dd}'
5772 | '\u{070f}'
5773 | '\u{08e2}'
5774 | '\u{180e}'
5775 | '\u{feff}'
5776 | '\u{fff9}'..='\u{fffb}'
5777 | '\u{0600}'..='\u{0605}'
5778 | '\u{0890}'..='\u{0891}'
5779 | '\u{200b}'..='\u{200f}'
5780 | '\u{202a}'..='\u{202e}'
5781 | '\u{2060}'..='\u{206f}'
5782 | '\u{110bd}'
5783 | '\u{110cd}'
5784 | '\u{13430}'..='\u{1343f}'
5785 | '\u{1bca0}'..='\u{1bca3}'
5786 | '\u{1d173}'..='\u{1d17a}'
5787 | '\u{e0001}'
5788 | '\u{e0020}'..='\u{e007f}'
5789 )
5790}
5791
5792fn format_between_timestamps(
5797 start: Option<UnixNanos>,
5798 end: Option<UnixNanos>,
5799 ts_now: UnixNanos,
5800) -> Option<String> {
5801 let (start, end) = match (start, end) {
5802 (None, None) => return None,
5803 (Some(s), Some(e)) => (s, e),
5804 (Some(s), None) => (s, ts_now),
5805 (None, Some(e)) => (UnixNanos::from(0), e),
5806 };
5807 let start_secs = start.as_u64() / 1_000_000_000;
5808 let end_secs = end.as_u64() / 1_000_000_000;
5809 Some(format!("{start_secs}-{end_secs}"))
5810}
5811
5812#[async_trait(?Send)]
5813impl ExecutionClient for LighterExecutionClient {
5814 fn is_connected(&self) -> bool {
5815 self.core.is_connected()
5816 }
5817
5818 fn client_id(&self) -> ClientId {
5819 self.core.client_id
5820 }
5821
5822 fn account_id(&self) -> AccountId {
5823 self.core.account_id
5824 }
5825
5826 fn venue(&self) -> Venue {
5827 self.core.venue
5828 }
5829
5830 fn oms_type(&self) -> OmsType {
5831 self.core.oms_type
5832 }
5833
5834 fn get_account(&self) -> Option<AccountAny> {
5835 self.core.cache().account_owned(&self.core.account_id)
5836 }
5837
5838 fn generate_account_state(
5839 &self,
5840 balances: Vec<AccountBalance>,
5841 margins: Vec<MarginBalance>,
5842 reported: bool,
5843 ts_event: UnixNanos,
5844 info: Option<Params>,
5845 ) -> anyhow::Result<()> {
5846 self.emitter
5847 .emit_account_state(balances, margins, reported, ts_event, info);
5848 Ok(())
5849 }
5850
5851 fn start(&mut self) -> anyhow::Result<()> {
5852 let sender = get_exec_event_sender();
5853 self.emitter.set_sender(sender);
5854 self.subscribe_applied_fills();
5855
5856 if self.core.is_started() {
5857 return Ok(());
5858 }
5859
5860 self.core.set_started();
5861
5862 log::info!(
5863 "Started Lighter execution client: client_id={}, account_id={}, environment={:?}, has_credentials={}",
5864 self.core.client_id,
5865 self.core.account_id,
5866 self.config.environment,
5867 self.has_credentials(),
5868 );
5869
5870 Ok(())
5871 }
5872
5873 fn stop(&mut self) -> anyhow::Result<()> {
5874 if self.core.is_stopped() && self.core.is_disconnected() && self.session_tasks_finished() {
5875 return Ok(());
5876 }
5877
5878 log::info!("Stopping Lighter execution client {}", self.core.client_id);
5879
5880 self.begin_session_shutdown();
5881
5882 self.core.set_stopped();
5883
5884 log::info!("Lighter execution client stopped");
5885 Ok(())
5886 }
5887
5888 fn reset(&mut self) -> anyhow::Result<()> {
5889 log::debug!("Resetting Lighter execution client {}", self.core.client_id);
5890 self.begin_session_shutdown();
5891 Ok(())
5892 }
5893
5894 fn dispose(&mut self) -> anyhow::Result<()> {
5895 log::debug!("Disposing Lighter execution client {}", self.core.client_id);
5896 self.stop()
5897 }
5898
5899 async fn connect(&mut self) -> anyhow::Result<()> {
5900 if self.core.is_connected() && self.pending_tasks.is_open() {
5901 return Ok(());
5902 }
5903
5904 if !self.has_credentials() {
5908 anyhow::bail!(
5909 "Lighter execution client requires credentials; \
5910 set private_key, account_index, and api_key_index in the config \
5911 or use the deployment-specific credential environment variables"
5912 );
5913 }
5914
5915 log::info!(
5916 "Connecting Lighter execution client {}",
5917 self.core.client_id
5918 );
5919
5920 if !self.session_tasks_finished() || !self.pending_tasks.is_open() {
5923 self.begin_session_shutdown();
5924 self.finish_session_shutdown().await?;
5925 }
5926
5927 if !self.pending_tasks.is_open() {
5928 self.pending_tasks
5929 .start_generation()
5930 .map_err(|e| anyhow::anyhow!("Failed to start Lighter task generation: {e}"))?;
5931 self.cancellation_token = self.pending_tasks.cancellation_token();
5932 }
5933
5934 let ws_client = self.ws_client.clone();
5935 let setup_guard = TaskGroupGuard::new(&[&self.pending_tasks], move || {
5936 ws_client.begin_shutdown();
5937 });
5938 self.auth_refresh_notify = Arc::new(tokio::sync::Notify::new());
5939 self.scheduled_cancel_notify = Arc::new(tokio::sync::Notify::new());
5940
5941 self.dispatch.account_streams_ready.reset();
5947 self.dispatch.clear_position_cache();
5948 self.dispatch.clear_account_state_cache();
5949
5950 self.ensure_instruments_initialized_async().await?;
5951 self.refresh_nonce().await?;
5952
5953 self.nonce_ready_connection_epoch
5955 .store(self.ws_client.connection_epoch(), Ordering::Release);
5956 let account_detail = self.fetch_account_detail().await;
5957
5958 if let Some(detail) = &account_detail {
5959 self.log_account_tier(detail);
5960
5961 let _ = tokio::time::timeout(
5962 REFERRAL_ATTRIBUTION_TIMEOUT,
5963 self.apply_referral_attribution(detail),
5964 )
5965 .await;
5966 }
5967
5968 if let Err(e) = self.submit_integrator_auto_approval().await {
5969 let is_unapproved = e.chain().any(|cause| {
5974 matches!(
5975 cause.downcast_ref::<LighterHttpError>(),
5976 Some(LighterHttpError::Venue { code: 21149, .. }),
5977 )
5978 });
5979
5980 if is_unapproved {
5981 self.nonce_ready_connection_epoch
5982 .store(NONCE_CONNECTION_EPOCH_UNAVAILABLE, Ordering::Release);
5983 return Err(e.context(
5984 "Lighter account is not integrator-approved (venue 21149); \
5985 orders cannot be placed",
5986 ));
5987 }
5988
5989 log::warn!("Lighter integrator approval failed; continuing startup: {e:?}");
5990 }
5991
5992 self.nonce_ready_connection_epoch
5993 .store(NONCE_CONNECTION_EPOCH_UNAVAILABLE, Ordering::Release);
5994
5995 if let Err(e) = self.sync_nonce_from_venue().await {
5996 log::debug!(
5997 "Failed to sync Lighter nonce after integrator approval; continuing startup: {e:?}"
5998 );
5999 }
6000
6001 self.subscribe_applied_fills();
6002
6003 if let Err(e) = self.spawn_ws_consumer().await {
6004 self.begin_session_shutdown();
6005 return match self.finish_session_shutdown().await {
6006 Ok(()) => Err(e),
6007 Err(shutdown_error) => Err(anyhow::anyhow!(
6008 "{e}; failed to roll back partial Lighter connection: {shutdown_error}"
6009 )),
6010 };
6011 }
6012 self.nonce_ready_connection_epoch
6013 .store(self.ws_client.connection_epoch(), Ordering::Release);
6014
6015 let ready = async {
6016 self.await_account_streams_ready(30.0).await?;
6017 self.start_scheduled_cancel().await
6018 }
6019 .await;
6020
6021 if let Err(e) = ready {
6022 log::warn!("Connect failed after WS started, tearing down: {e}");
6023 self.begin_session_shutdown();
6024
6025 if let Err(shutdown_error) = self.finish_session_shutdown().await {
6026 log::warn!("Failed to finish partial Lighter connection: {shutdown_error}");
6027 }
6028
6029 return Err(e);
6030 }
6031
6032 setup_guard.disarm();
6033 self.core.set_connected();
6034
6035 log::info!("Connected: client_id={}", self.core.client_id);
6036 Ok(())
6037 }
6038
6039 async fn disconnect(&mut self) -> anyhow::Result<()> {
6040 if self.core.is_disconnected() && self.session_tasks_finished() {
6041 return Ok(());
6042 }
6043
6044 log::info!(
6045 "Disconnecting Lighter execution client {}",
6046 self.core.client_id
6047 );
6048
6049 self.begin_session_shutdown();
6050 let tasks_result = self.finish_session_shutdown().await;
6051
6052 self.core.set_disconnected();
6053
6054 log::info!("Disconnected: client_id={}", self.core.client_id);
6055 tasks_result
6056 }
6057
6058 fn submit_order(&self, cmd: SubmitOrder) -> anyhow::Result<()> {
6059 let credential = self.credential.as_ref().ok_or_else(|| {
6060 anyhow::anyhow!("Lighter execution client cannot submit without credentials")
6061 })?;
6062
6063 let order = self.core.cache().try_order_owned(&cmd.client_order_id)?;
6064
6065 if order.is_closed() {
6066 log::warn!("Cannot submit closed order {}", order.client_order_id());
6067 return Ok(());
6068 }
6069
6070 let cached_instrument = self
6071 .core
6072 .cache()
6073 .instrument(&order.instrument_id())
6074 .cloned();
6075
6076 if let Some(reason) = local_submit_denial_reason(&order, cached_instrument.as_ref()) {
6077 self.emitter.emit_order_denied(&order, &reason.to_string());
6078 return Ok(());
6079 }
6080
6081 let slippage_bps = self.resolve_slippage_bps(cmd.params.as_ref());
6082
6083 let plan = match self.prepare_create_order_plan(&order, slippage_bps, cmd.params.as_ref()) {
6084 Ok(plan) => plan,
6085 Err(e) => {
6086 let reason = OrderDeniedReason::ValidationFailed {
6087 detail: format!("Lighter submit_order failed: {e}"),
6088 };
6089 self.emitter.emit_order_denied(&order, &reason.to_string());
6090 return Ok(());
6091 }
6092 };
6093
6094 if let Err(e) = self.dispatch_create_order_plan(&plan, credential) {
6095 let reason = OrderDeniedReason::SubmitFailed {
6096 detail: format!("Lighter submit_order failed: {e}"),
6097 };
6098 self.emitter.emit_order_denied(&order, &reason.to_string());
6099 }
6100
6101 Ok(())
6102 }
6103
6104 fn submit_order_list(&self, cmd: SubmitOrderList) -> anyhow::Result<()> {
6105 let credential = self.credential.as_ref().ok_or_else(|| {
6106 anyhow::anyhow!("Lighter execution client cannot submit without credentials")
6107 })?;
6108
6109 if cmd.order_list.client_order_ids.is_empty() {
6110 log::debug!("submit_order_list called with empty order list");
6111 return Ok(());
6112 }
6113
6114 let orders = self.core.get_orders_for_list(&cmd.order_list)?;
6115
6116 if orders.iter().any(is_grouped_order) {
6117 let result = self
6118 .prepare_create_group_plan(
6119 &orders,
6120 self.resolve_slippage_bps(cmd.params.as_ref()),
6121 cmd.params.as_ref(),
6122 )
6123 .and_then(|plan| {
6124 let context = self.fanout_dispatch_context(credential)?;
6125 self.spawn_task("submit_grouped_orders", async move {
6126 context
6127 .dispatch_batch_plans(vec![BatchOrderPlan::CreateGroup(plan)])
6128 .await;
6129 Ok(())
6130 });
6131
6132 Ok(())
6133 });
6134
6135 if let Err(e) = result {
6136 let reason = grouped_submit_denial_reason(&e);
6137 for order in &orders {
6138 self.emitter.emit_order_denied(order, &reason);
6139 }
6140 }
6141 return Ok(());
6142 }
6143
6144 let slippage_bps = self.resolve_slippage_bps(cmd.params.as_ref());
6145 let mut plans = Vec::with_capacity(orders.len());
6146
6147 for order in orders {
6148 if order.is_closed() {
6149 log::warn!("Cannot submit closed order {}", order.client_order_id());
6150 continue;
6151 }
6152
6153 let cached_instrument = self
6154 .core
6155 .cache()
6156 .instrument(&order.instrument_id())
6157 .cloned();
6158
6159 if let Some(reason) = local_submit_denial_reason(&order, cached_instrument.as_ref()) {
6160 self.emitter.emit_order_denied(&order, &reason.to_string());
6161 continue;
6162 }
6163
6164 match self.prepare_create_order_plan(&order, slippage_bps, cmd.params.as_ref()) {
6165 Ok(plan) => plans.push(plan),
6166 Err(e) => {
6167 let reason = OrderDeniedReason::ValidationFailed {
6168 detail: format!("Lighter submit_order_list failed: {e}"),
6169 }
6170 .to_string();
6171
6172 self.emitter.emit_order_denied(&order, &reason);
6173 }
6174 }
6175 }
6176
6177 if plans.is_empty() {
6178 log::warn!(
6179 "Lighter submit_order_list: no supported orders to dispatch for {}",
6180 cmd.order_list.id,
6181 );
6182 return Ok(());
6183 }
6184
6185 let context = self.fanout_dispatch_context(credential)?;
6186 self.spawn_task("submit_order_list", async move {
6187 context
6188 .dispatch_batch_plans(
6189 plans
6190 .into_iter()
6191 .map(|plan| BatchOrderPlan::Create(Box::new(plan)))
6192 .collect(),
6193 )
6194 .await;
6195 Ok(())
6196 });
6197
6198 Ok(())
6199 }
6200
6201 fn modify_order(&self, cmd: ModifyOrder) -> anyhow::Result<()> {
6202 let credential = self.credential.as_ref().ok_or_else(|| {
6203 anyhow::anyhow!("Lighter execution client cannot modify without credentials")
6204 })?;
6205 self.dispatch_signed_modify_order(&cmd, credential);
6206 Ok(())
6207 }
6208
6209 fn batch_modify_orders(&self, cmd: BatchModifyOrders) -> anyhow::Result<()> {
6210 let credential = self.credential.as_ref().ok_or_else(|| {
6211 anyhow::anyhow!("Lighter execution client cannot modify without credentials")
6212 })?;
6213
6214 let context = self.fanout_dispatch_context(credential)?;
6215 let mut plans = Vec::with_capacity(cmd.modifies.len());
6216 for modify in &cmd.modifies {
6217 if !self.admit_modify_order(modify) {
6218 continue;
6219 }
6220
6221 match self.prepare_modify_order_plan(modify) {
6222 Ok(plan) => plans.push(BatchOrderPlan::Modify(plan)),
6223 Err(e) => {
6224 let mut events = self.dispatch.order_events.lock();
6225 self.dispatch.clear_pending_modify_if_command(
6226 &modify.client_order_id,
6227 modify.command_id,
6228 );
6229 self.emitter.emit_order_modify_rejected_event(
6230 modify.strategy_id,
6231 modify.instrument_id,
6232 modify.client_order_id,
6233 modify.venue_order_id,
6234 &format!("Lighter modify_order failed: {e}"),
6235 self.clock.get_time_ns(),
6236 );
6237
6238 if let Some(deferred) = events.remove(&modify.client_order_id) {
6239 flush_modify_events(
6240 modify.client_order_id,
6241 deferred,
6242 &self.dispatch,
6243 &self.emitter,
6244 &mut events,
6245 );
6246 }
6247 }
6248 }
6249 }
6250
6251 if !plans.is_empty() {
6252 self.spawn_task("batch_modify_orders", async move {
6253 context.dispatch_batch_plans(plans).await;
6254 Ok(())
6255 });
6256 }
6257
6258 Ok(())
6259 }
6260
6261 fn cancel_order(&self, cmd: CancelOrder) -> anyhow::Result<()> {
6262 let credential = self.credential.as_ref().ok_or_else(|| {
6263 anyhow::anyhow!("Lighter execution client cannot cancel without credentials")
6264 })?;
6265
6266 if let Some(plans) = self.prepare_oco_cancel_plans(&cmd) {
6267 let context = self.fanout_dispatch_context(credential)?;
6268
6269 let plans = plans
6270 .into_iter()
6271 .map(|plan| {
6272 let emit = self.can_emit_order_cancel_rejected(&plan.client_order_id);
6273 self.dispatch.set_pending_order_action(
6274 plan.client_order_id,
6275 PendingOrderAction::cancel(emit),
6276 );
6277 BatchOrderPlan::Cancel(plan, emit)
6278 })
6279 .collect();
6280
6281 self.spawn_task("cancel_oco_orders", async move {
6282 context.dispatch_batch_plans(plans).await;
6283 Ok(())
6284 });
6285
6286 return Ok(());
6287 }
6288
6289 self.dispatch_signed_cancel_order(&cmd, credential);
6290 Ok(())
6291 }
6292
6293 fn cancel_all_orders(&self, cmd: CancelAllOrders) -> anyhow::Result<()> {
6294 let cancels: Vec<_> = self
6297 .core
6298 .cache()
6299 .orders_open(
6300 None,
6301 Some(&cmd.instrument_id),
6302 None,
6303 Some(&self.core.account_id),
6304 cmd.order_side,
6305 )
6306 .into_iter()
6307 .map(|order| {
6308 cancel_order_from_cancel_all(&cmd, order.client_order_id(), order.strategy_id())
6309 })
6310 .collect();
6311
6312 if cancels.is_empty() {
6313 return Ok(());
6314 }
6315
6316 self.dispatch_cancel_orders(&cancels)
6317 }
6318
6319 fn batch_cancel_orders(&self, cmd: BatchCancelOrders) -> anyhow::Result<()> {
6320 self.dispatch_cancel_orders(&cmd.cancels)
6321 }
6322
6323 fn query_account(&self, _cmd: QueryAccount) -> anyhow::Result<()> {
6324 let cached = self.dispatch.snapshot_account_state();
6330 match cached {
6331 Some(state) => {
6332 log::debug!("Lighter query_account replaying cached AccountState");
6333 self.emitter.send_account_state(state);
6334 }
6335 None => {
6336 log::warn!(
6337 "Lighter query_account: no AccountState cached yet \
6338 (account_all_assets stream has not pushed since connect)",
6339 );
6340 }
6341 }
6342 Ok(())
6343 }
6344
6345 fn query_order(&self, cmd: QueryOrder) -> anyhow::Result<()> {
6346 let credential = self
6347 .credential
6348 .as_ref()
6349 .ok_or_else(|| anyhow::anyhow!("Lighter query_order requires credentials"))?
6350 .clone();
6351 let registry = Arc::clone(&self.registry);
6352 let http_client = self.http_client.clone();
6353 let emitter = self.emitter.clone();
6354 let core_account_id = self.core.account_id;
6355 let dispatch = self.dispatch.clone();
6356 let clock = self.clock;
6357
6358 if let Some(order) = self.core.cache().order_owned(&cmd.client_order_id) {
6359 self.dispatch.seed_order_restoration(order);
6360 }
6361
6362 let _ = order_shape_pending(&self.core, &self.dispatch, &cmd.client_order_id);
6363 let child_context = self.child_probe_context();
6364 self.spawn_task("query_order", async move {
6365 let lookup = OrderReportLookup::new(
6366 ®istry,
6367 &dispatch,
6368 Some(cmd.instrument_id),
6369 Some(&cmd.client_order_id),
6370 cmd.venue_order_id.as_ref(),
6371 )?;
6372
6373 let report = if let Some(order) =
6374 collect_order(&http_client, ®istry, &credential, &lookup).await?
6375 {
6376 if !dispatch.restore_cached_order(
6377 VenueOrderId::new(&order.order_id),
6378 order.client_order_index,
6379 cmd.instrument_id,
6380 ) {
6381 return Ok(());
6382 }
6383
6384 if let Some(context) = &child_context {
6385 recover_report_fills(&order, context).await?;
6386 }
6387
6388 let parsed = lookup.parse(
6389 &order,
6390 ®istry,
6391 core_account_id,
6392 clock.get_time_ns(),
6393 &dispatch,
6394 )?;
6395 let report = lookup.finish(&dispatch, parsed);
6396 if dispatch
6397 .order_identity(&cmd.client_order_id)
6398 .is_some_and(|identity| identity.child_fill_baseline().is_some())
6399 {
6400 if let Some(context) = &child_context {
6401 spawn_child_recovery(cmd.client_order_id, context);
6402 }
6403
6404 return Ok(());
6405 }
6406
6407 Some(report)
6408 } else {
6409 None
6410 };
6411
6412 let events = dispatch.order_events.lock();
6413 if dispatch
6414 .terminals_pending
6415 .lock()
6416 .contains_key(&cmd.client_order_id)
6417 || events.get(&cmd.client_order_id).is_some_and(|deferred| {
6418 deferred
6419 .order
6420 .as_ref()
6421 .is_some_and(|order| order.status.is_terminal())
6422 })
6423 || dispatch
6424 .order_identity(&cmd.client_order_id)
6425 .is_some_and(|identity| {
6426 position_fills_pending(&identity, &events, &dispatch)
6427 || dispatch.any_order_identity(|_, other, _| {
6428 other.strategy_id == identity.strategy_id
6429 && other.instrument_id == identity.instrument_id
6430 && other.has_publication()
6431 })
6432 })
6433 || dispatch.has_pending_modify(&cmd.client_order_id)
6434 || dispatch
6435 .order_identity(&cmd.client_order_id)
6436 .is_some_and(|identity| identity.has_publication())
6437 {
6438 return Ok(());
6439 }
6440
6441 match report {
6442 Some(report) => {
6443 if dispatch
6444 .order_identity(&cmd.client_order_id)
6445 .is_some_and(|identity| !identity.report_fills_covered(&report, &[]))
6446 {
6447 return Ok(());
6448 }
6449
6450 log::debug!(
6451 "Lighter query_order returning report for {}",
6452 cmd.client_order_id
6453 );
6454 dispatch.seed_accepted_from_report(&report);
6455 emitter.send_order_status_report(report);
6456 }
6457 None => {
6458 log::warn!(
6459 "Lighter query_order: no order found for {}",
6460 cmd.client_order_id,
6461 );
6462 }
6463 }
6464 Ok(())
6465 });
6466 Ok(())
6467 }
6468
6469 fn generate_order_status_report_task(
6470 &self,
6471 cmd: &GenerateOrderStatusReport,
6472 ) -> Option<ExecutionReportTask<Option<OrderStatusReport>>> {
6473 let credential = self.credential.clone();
6474 let http = self.http_client.clone();
6475 let registry = Arc::clone(&self.registry);
6476 let account_id = self.core.account_id;
6477 let clock = self.clock;
6478 let core = self.core.clone();
6479 let dispatch = self.dispatch.clone();
6480 let cancellation_token = self.cancellation_token.clone();
6481 let command = cmd.clone();
6482 let child_context = self.child_probe_context();
6483 let collection_context = child_context.clone();
6484 let lookup_registry = Arc::clone(®istry);
6485
6486 let lookup = OrderReportLookup::new(
6487 ®istry,
6488 &dispatch,
6489 cmd.instrument_id,
6490 cmd.client_order_id.as_ref(),
6491 cmd.venue_order_id.as_ref(),
6492 );
6493 Some(ExecutionReportTask::new(
6494 async move {
6495 let Some(credential) = credential else {
6496 log::warn!("Lighter generate_order_status_report: no credentials");
6497 return Ok(None);
6498 };
6499
6500 if command.client_order_id.is_none() && command.venue_order_id.is_none() {
6501 log::warn!(
6502 "Lighter generate_order_status_report: must supply client_order_id or venue_order_id"
6503 );
6504 return Ok(None);
6505 }
6506
6507 let lookup = lookup?;
6508 let order = collect_order(&http, &lookup_registry, &credential, &lookup).await?;
6509 if let Some(order) = &order
6510 && order.status == LighterOrderStatus::Filled
6511 && let Some(context) = &collection_context
6512 {
6513 recover_report_fills(order, context).await?;
6514 }
6515
6516 Ok(order.map(|order| (lookup, order)))
6517 },
6518 move |collected| {
6519 anyhow::ensure!(
6520 !cancellation_token.is_cancelled(),
6521 "Lighter report session stopped"
6522 );
6523
6524 let report = collected
6525 .map(|(lookup, order)| -> anyhow::Result<OrderStatusReport> {
6526 let parsed = lookup.parse(
6527 &order,
6528 ®istry,
6529 account_id,
6530 clock.get_time_ns(),
6531 &dispatch,
6532 )?;
6533 let report = &parsed.0;
6534 let pending = core
6535 .cache()
6536 .client_order_id(&report.venue_order_id)
6537 .copied()
6538 .is_some_and(|cloid| dispatch.has_pending_modify(&cloid));
6539
6540 if !pending {
6541 restore_reconciled_order(
6542 &core,
6543 &dispatch,
6544 report.venue_order_id,
6545 order.client_order_index,
6546 report.order_status.is_closed(),
6547 );
6548 dispatch.record_order_reduction(
6549 order.client_order_index,
6550 report.venue_order_id,
6551 order.initial_base_amount - report.quantity.as_decimal(),
6552 );
6553 }
6554
6555 let report = lookup.finish(&dispatch, parsed);
6556 if let Some(context) = &child_context {
6557 request_child_recovery(&core, context, &report);
6558 }
6559
6560 ensure_order_report_ready(&core, &dispatch, &report)?;
6561 Ok(report)
6562 })
6563 .transpose()?;
6564
6565 if let Some(report) = &report {
6566 dispatch.seed_accepted_from_report(report);
6567 }
6568
6569 Ok(report)
6570 },
6571 ))
6572 }
6573
6574 fn generate_order_status_reports_task(
6575 &self,
6576 cmd: &GenerateOrderStatusReports,
6577 ) -> Option<ExecutionReportTask<Vec<OrderStatusReport>>> {
6578 let client = self.report_client();
6579 let core = self.core.clone();
6580 let dispatch = self.dispatch.clone();
6581 let cancellation_token = self.cancellation_token.clone();
6582 let child_context = self.child_probe_context();
6583 let command = cmd.clone();
6584 let collection_command = command.clone();
6585 let collection_context = child_context.clone();
6586 Some(ExecutionReportTask::new(
6587 async move {
6588 let sweep = client
6589 .collect_order_status_reports(&collection_command)
6590 .await;
6591
6592 if let Some(context) = &collection_context {
6593 recover_filled_reports(&sweep, context).await;
6594 }
6595
6596 Ok(sweep)
6597 },
6598 move |sweep| {
6599 anyhow::ensure!(
6600 !cancellation_token.is_cancelled(),
6601 "Lighter report session stopped"
6602 );
6603
6604 if let Some(context) = &child_context {
6605 request_child_report_recoveries(&core, context, &sweep);
6606 }
6607
6608 finish_order_reports(&core, &dispatch, sweep, &command, None)
6609 },
6610 ))
6611 }
6612
6613 fn generate_fill_reports_task(
6614 &self,
6615 cmd: &GenerateFillReports,
6616 ) -> Option<ExecutionReportTask<Vec<FillReport>>> {
6617 let client = self.report_client();
6618 let instruments = self.report_instruments(cmd.instrument_id);
6619 let core = self.core.clone();
6620 let dispatch = self.dispatch.clone();
6621 let cancellation_token = self.cancellation_token.clone();
6622 let child_context = self.child_probe_context();
6623 let command = cmd.clone();
6624 let cmd_log_level = cmd.log_receipt_level;
6625 Some(ExecutionReportTask::new(
6626 async move {
6627 let mut markets = BTreeSet::new();
6628 let result = client
6629 .collect_fill_reports(&command, &instruments, &mut markets)
6630 .await;
6631 Ok((markets, result))
6632 },
6633 move |(markets, result)| {
6634 anyhow::ensure!(
6635 !cancellation_token.is_cancelled(),
6636 "Lighter report session stopped"
6637 );
6638 let reports = finish_fill_reports(&core, &dispatch, markets, result)?.reports;
6639
6640 if let Some(context) = &child_context {
6641 spawn_child_recoveries(context);
6642 }
6643
6644 Self::log_report_receipt(reports.len(), "FillReport", cmd_log_level);
6645 Ok(reports)
6646 },
6647 ))
6648 }
6649
6650 fn generate_position_status_reports_task(
6651 &self,
6652 cmd: &GeneratePositionStatusReports,
6653 ) -> Option<ExecutionReportTask<Vec<PositionStatusReport>>> {
6654 let core = self.core.clone();
6655 let dispatch = self.dispatch.clone();
6656 let cancellation_token = self.cancellation_token.clone();
6657 let registry = Arc::clone(&self.registry);
6658 let command = cmd.clone();
6659 Some(ExecutionReportTask::new(async { Ok(()) }, move |()| {
6660 anyhow::ensure!(
6661 !cancellation_token.is_cancelled(),
6662 "Lighter report session stopped"
6663 );
6664 let (reports, complete, _) =
6665 cached_position_reports(&core, &dispatch, ®istry, &command)?;
6666 anyhow::ensure!(
6667 complete,
6668 "Lighter position snapshot does not cover the requested instrument scope"
6669 );
6670 Self::log_report_receipt(
6671 reports.len(),
6672 "PositionStatusReport",
6673 command.log_receipt_level,
6674 );
6675 Ok(reports)
6676 }))
6677 }
6678
6679 async fn generate_order_status_report(
6680 &self,
6681 cmd: &GenerateOrderStatusReport,
6682 ) -> anyhow::Result<Option<OrderStatusReport>> {
6683 let Some(credential) = &self.credential else {
6684 log::warn!("Lighter generate_order_status_report: no credentials");
6685 return Ok(None);
6686 };
6687
6688 if cmd.client_order_id.is_none() && cmd.venue_order_id.is_none() {
6689 log::warn!(
6690 "Lighter generate_order_status_report: must supply client_order_id or venue_order_id",
6691 );
6692 return Ok(None);
6693 }
6694
6695 let lookup = OrderReportLookup::new(
6696 &self.registry,
6697 &self.dispatch,
6698 cmd.instrument_id,
6699 cmd.client_order_id.as_ref(),
6700 cmd.venue_order_id.as_ref(),
6701 )?;
6702
6703 let report = if let Some(order) =
6704 collect_order(&self.http_client, &self.registry, credential, &lookup).await?
6705 {
6706 if order.status == LighterOrderStatus::Filled
6707 && let Some(context) = self.child_probe_context()
6708 {
6709 recover_report_fills(&order, &context).await?;
6710 }
6711
6712 let parsed = lookup.parse(
6713 &order,
6714 &self.registry,
6715 self.core.account_id,
6716 self.clock.get_time_ns(),
6717 &self.dispatch,
6718 )?;
6719
6720 if !self
6721 .core
6722 .cache()
6723 .client_order_id(&parsed.0.venue_order_id)
6724 .is_some_and(|cloid| self.dispatch.has_pending_modify(cloid))
6725 {
6726 restore_reconciled_order(
6727 &self.core,
6728 &self.dispatch,
6729 parsed.0.venue_order_id,
6730 order.client_order_index,
6731 parsed.0.order_status.is_closed(),
6732 );
6733 self.dispatch.record_order_reduction(
6734 order.client_order_index,
6735 parsed.0.venue_order_id,
6736 order.initial_base_amount - parsed.0.quantity.as_decimal(),
6737 );
6738 }
6739
6740 let report = lookup.finish(&self.dispatch, parsed);
6741 if let Some(context) = self.child_probe_context() {
6742 request_child_recovery(&self.core, &context, &report);
6743 }
6744
6745 ensure_order_report_ready(&self.core, &self.dispatch, &report)?;
6746 Some(report)
6747 } else {
6748 None
6749 };
6750
6751 if let Some(report) = &report {
6752 self.dispatch.seed_accepted_from_report(report);
6753 }
6754
6755 Ok(report)
6756 }
6757
6758 async fn generate_order_status_reports(
6759 &self,
6760 cmd: &GenerateOrderStatusReports,
6761 ) -> anyhow::Result<Vec<OrderStatusReport>> {
6762 self.collect_order_reports(cmd, None).await
6763 }
6764
6765 async fn generate_fill_reports(
6766 &self,
6767 cmd: GenerateFillReports,
6768 ) -> anyhow::Result<Vec<FillReport>> {
6769 let reports = self.paginate_fill_reports(&cmd).await?.reports;
6770 Self::log_report_receipt(reports.len(), "FillReport", cmd.log_receipt_level);
6771 Ok(reports)
6772 }
6773
6774 async fn generate_position_status_reports(
6775 &self,
6776 cmd: &GeneratePositionStatusReports,
6777 ) -> anyhow::Result<Vec<PositionStatusReport>> {
6778 let (reports, complete, _) = self.cached_position_reports(cmd)?;
6779 anyhow::ensure!(
6780 complete,
6781 "Lighter position snapshot does not cover the requested instrument scope",
6782 );
6783 Self::log_report_receipt(reports.len(), "PositionStatusReport", cmd.log_receipt_level);
6784 Ok(reports)
6785 }
6786
6787 async fn generate_mass_status(
6788 &self,
6789 lookback_mins: Option<u64>,
6790 ) -> anyhow::Result<Option<ExecutionMassStatus>> {
6791 let ts_init = self.clock.get_time_ns();
6792
6793 let lookback_start = lookback_mins
6796 .map(DurationNanos::try_from_mins)
6797 .transpose()?
6798 .map(|lookback| ts_init.saturating_sub(lookback));
6799
6800 let mut order_cmd = GenerateOrderStatusReports::new(
6805 UUID4::new(),
6806 ts_init,
6807 false,
6808 None,
6809 lookback_start,
6810 None,
6811 None,
6812 None,
6813 );
6814 order_cmd.log_receipt_level = LogLevel::Debug;
6815 let fill_cmd = GenerateFillReports::new(
6816 UUID4::new(),
6817 ts_init,
6818 None,
6819 None,
6820 lookback_start,
6821 None,
6822 None,
6823 None,
6824 );
6825 let position_cmd =
6826 GeneratePositionStatusReports::new(UUID4::new(), ts_init, None, None, None, None, None);
6827
6828 let (mut fill_reports, fill_reports_complete, withheld_fills) =
6829 fill_reports_for_mass_status(self.paginate_fill_reports(&fill_cmd).await)
6830 .context("Lighter fill reconciliation failed")?;
6831 Self::log_report_receipt(fill_reports.len(), "FillReport", fill_cmd.log_receipt_level);
6832
6833 let mut withheld_instruments = withheld_fills;
6836 let order_result = self
6837 .collect_order_reports(&order_cmd, Some(&fill_reports))
6838 .await;
6839
6840 let (mut order_reports, mut reports_complete) = match order_result {
6841 Ok(reports) => (reports, true),
6842 Err(e) => {
6843 withheld_instruments.extend(withheld_order_instruments(&e));
6844 log::warn!(
6845 "Lighter order report generation failed: {}",
6846 scrub_auth(&format!("{e:#}")),
6847 );
6848 let partial = partial_order_reports(&e);
6849 let reports = if partial.is_empty() {
6850 let mut active_cmd = GenerateOrderStatusReports::new(
6851 UUID4::new(),
6852 ts_init,
6853 true,
6854 None,
6855 None,
6856 None,
6857 None,
6858 None,
6859 );
6860 active_cmd.log_receipt_level = LogLevel::Debug;
6861 let active_result = self
6862 .collect_order_reports(&active_cmd, Some(&fill_reports))
6863 .await;
6864
6865 match active_result {
6866 Ok(reports) => reports,
6867 Err(e) => {
6868 withheld_instruments.extend(withheld_order_instruments(&e));
6869 log::warn!(
6870 "Lighter active order report generation incomplete: {}",
6871 scrub_auth(&format!("{e:#}")),
6872 );
6873 partial_order_reports(&e)
6874 }
6875 }
6876 } else {
6877 partial
6878 };
6879 (reports, false)
6880 }
6881 };
6882
6883 reports_complete &= fill_reports_complete;
6884 let mut reported_orders: AHashSet<VenueOrderId> = order_reports
6885 .iter()
6886 .map(|report| report.venue_order_id)
6887 .collect();
6888 let mut fill_markets: Vec<i64> = fill_reports
6889 .iter()
6890 .filter(|report| !reported_orders.contains(&report.venue_order_id))
6891 .filter_map(|report| self.registry.market_index(&report.instrument_id))
6892 .collect::<AHashSet<_>>()
6893 .into_iter()
6894 .collect();
6895 fill_markets.sort_unstable();
6896
6897 for market_index in fill_markets {
6898 let Some(instrument_id) = self.registry.instrument_id(market_index) else {
6899 reports_complete = false;
6900 continue;
6901 };
6902 let mut order_cmd = GenerateOrderStatusReports::new(
6903 UUID4::new(),
6904 ts_init,
6905 false,
6906 Some(instrument_id),
6907 lookback_start,
6908 None,
6909 None,
6910 None,
6911 );
6912 order_cmd.log_receipt_level = LogLevel::Debug;
6913 let order_result = self
6914 .collect_order_reports(&order_cmd, Some(&fill_reports))
6915 .await;
6916 reports_complete &= order_result.is_ok();
6917 let reports = match order_result {
6918 Ok(reports) => reports,
6919 Err(e) => {
6920 withheld_instruments.extend(withheld_order_instruments(&e));
6921 log::warn!(
6922 "Lighter order report generation failed: {}",
6923 scrub_auth(&format!("{e:#}")),
6924 );
6925 partial_order_reports(&e)
6926 }
6927 };
6928 reported_orders.extend(reports.iter().map(|report| report.venue_order_id));
6929 order_reports.extend(reports);
6930 }
6931
6932 if fill_reports
6933 .iter()
6934 .any(|report| !reported_orders.contains(&report.venue_order_id))
6935 {
6936 reports_complete = false;
6937 }
6938
6939 for fill_report in &mut fill_reports {
6940 if let Some(client_order_id) = order_reports
6941 .iter()
6942 .find(|order_report| order_report.venue_order_id == fill_report.venue_order_id)
6943 .and_then(|order_report| order_report.client_order_id)
6944 {
6945 fill_report.client_order_id = Some(client_order_id);
6946 }
6947 }
6948
6949 self.publish_terminal_fill_bundles(&mut order_reports, &fill_reports)?;
6950
6951 let position_result = self.cached_position_reports(&position_cmd);
6952
6953 let (mut position_reports, mut position_coverage) = match position_result {
6954 Ok((reports, complete, coverage)) => {
6955 reports_complete &= complete;
6956 (reports, coverage)
6957 }
6958 Err(e) => {
6959 reports_complete = false;
6960 log::warn!("Lighter position report generation failed: {e:#}");
6961 (Vec::new(), None)
6962 }
6963 };
6964
6965 position_reports.retain(|report| !withheld_instruments.contains(&report.instrument_id));
6966
6967 if let Some(skipped) = &mut position_coverage {
6968 skipped.extend(
6969 withheld_instruments
6970 .iter()
6971 .filter_map(|id| self.registry.market_index(id)),
6972 );
6973 }
6974
6975 Self::log_report_receipt(
6976 position_reports.len(),
6977 "PositionStatusReport",
6978 position_cmd.log_receipt_level,
6979 );
6980
6981 if lookback_start.is_some() {
6982 let touched_instruments: AHashSet<InstrumentId> = order_reports
6983 .iter()
6984 .map(|report| report.instrument_id)
6985 .chain(fill_reports.iter().map(|report| report.instrument_id))
6986 .collect();
6987 let position_instruments: AHashSet<InstrumentId> = position_reports
6988 .iter()
6989 .map(|report| report.instrument_id)
6990 .collect();
6991 let cache = self.core.cache();
6992 let mut touched_instruments: Vec<InstrumentId> =
6993 touched_instruments.into_iter().collect();
6994 touched_instruments.sort_unstable();
6995
6996 for instrument_id in touched_instruments {
6997 let Some(instrument) = cache.instrument(&instrument_id) else {
6998 reports_complete = false;
6999 continue;
7000 };
7001
7002 if !matches!(instrument, InstrumentAny::CryptoPerpetual(_)) {
7003 continue;
7004 }
7005 let Some(market_id) = self.registry.market_index(&instrument_id) else {
7006 reports_complete = false;
7007 continue;
7008 };
7009
7010 if !position_coverage
7011 .as_ref()
7012 .is_some_and(|skipped| !skipped.contains(&market_id))
7013 {
7014 reports_complete = false;
7015 continue;
7016 }
7017
7018 if position_instruments.contains(&instrument_id) {
7019 continue;
7020 }
7021
7022 position_reports.push(PositionStatusReport::new(
7023 self.core.account_id,
7024 instrument_id,
7025 PositionSide::Flat,
7026 Quantity::zero(instrument.size_precision()),
7027 ts_init,
7028 ts_init,
7029 Some(UUID4::new()),
7030 None,
7031 None,
7032 ));
7033 }
7034 }
7035
7036 let mut mass_status = ExecutionMassStatus::new(
7037 self.core.client_id,
7038 self.core.account_id,
7039 self.core.venue,
7040 ts_init,
7041 None,
7042 );
7043 mass_status.set_report_window(lookback_start, reports_complete);
7044 mass_status.add_order_reports(order_reports);
7045 mass_status.add_fill_reports(fill_reports);
7046 mass_status.add_position_reports(position_reports);
7047
7048 log::debug!(
7049 "Generated Lighter mass status: {} orders, {} fills, {} positions",
7050 mass_status.order_reports().len(),
7051 mass_status.fill_reports().len(),
7052 mass_status.position_reports().len(),
7053 );
7054
7055 Ok(Some(mass_status))
7056 }
7057}
7058
7059#[derive(Debug)]
7060struct AppliedFillSubscription(TypedHandler<OrderEventAny>);
7061
7062impl Drop for AppliedFillSubscription {
7063 fn drop(&mut self) {
7064 msgbus::unsubscribe_order_events(ORDER_EVENTS_PATTERN.into(), &self.0);
7065 }
7066}
7067
7068struct FillSweep {
7072 reports: Vec<FillReport>,
7073 covers_window: bool,
7074 withheld_instruments: AHashSet<InstrumentId>,
7075}
7076
7077#[derive(Debug, thiserror::Error)]
7078#[error("incomplete Lighter order reports: {detail}")]
7079struct IncompleteOrderReports {
7080 reports: Vec<OrderStatusReport>,
7081 withheld_instruments: AHashSet<InstrumentId>,
7082 detail: String,
7083}
7084
7085fn incomplete_order_reports(
7086 reports: Vec<OrderStatusReport>,
7087 detail: impl Into<String>,
7088 withheld_instruments: AHashSet<InstrumentId>,
7089) -> anyhow::Error {
7090 anyhow::Error::new(IncompleteOrderReports {
7091 reports,
7092 withheld_instruments,
7093 detail: detail.into(),
7094 })
7095}
7096
7097fn partial_order_reports(error: &anyhow::Error) -> Vec<OrderStatusReport> {
7098 error
7099 .downcast_ref::<IncompleteOrderReports>()
7100 .map(|incomplete| incomplete.reports.clone())
7101 .unwrap_or_default()
7102}
7103
7104fn withheld_order_instruments(error: &anyhow::Error) -> AHashSet<InstrumentId> {
7105 error
7106 .downcast_ref::<IncompleteOrderReports>()
7107 .map(|incomplete| incomplete.withheld_instruments.clone())
7108 .unwrap_or_default()
7109}
7110
7111fn is_commission_error(error: &anyhow::Error) -> bool {
7112 error
7113 .chain()
7114 .any(|cause| cause.downcast_ref::<LighterCommissionError>().is_some())
7115}
7116
7117fn fill_reports_for_mass_status(
7118 result: anyhow::Result<FillSweep>,
7119) -> anyhow::Result<(Vec<FillReport>, bool, AHashSet<InstrumentId>)> {
7120 match result {
7121 Ok(sweep) => Ok((
7122 sweep.reports,
7123 sweep.covers_window,
7124 sweep.withheld_instruments,
7125 )),
7126 Err(e) if is_commission_error(&e) => Err(e),
7127 Err(e) => {
7128 log::warn!(
7129 "Lighter fill report generation failed: {}",
7130 scrub_auth(&format!("{e:#}")),
7131 );
7132 Ok((Vec::new(), false, AHashSet::new()))
7133 }
7134 }
7135}
7136
7137impl LighterExecutionClient {
7138 async fn collect_order_reports(
7139 &self,
7140 cmd: &GenerateOrderStatusReports,
7141 fills: Option<&[FillReport]>,
7142 ) -> anyhow::Result<Vec<OrderStatusReport>> {
7143 let sweep = self.report_client().collect_order_status_reports(cmd).await;
7144 if let Some(context) = self.child_probe_context() {
7145 request_child_report_recoveries(&self.core, &context, &sweep);
7146 if fills.is_none() {
7147 recover_filled_reports(&sweep, &context).await;
7148 }
7149 }
7150
7151 finish_order_reports(&self.core, &self.dispatch, sweep, cmd, fills)
7152 }
7153
7154 fn publish_terminal_fill_bundles(
7155 &self,
7156 orders: &mut Vec<OrderStatusReport>,
7157 fills: &[FillReport],
7158 ) -> anyhow::Result<()> {
7159 let mut bundled = AHashSet::new();
7160
7161 for report in orders.iter() {
7162 if !matches!(
7163 report.order_status,
7164 OrderStatus::Canceled | OrderStatus::Expired
7165 ) || report
7166 .client_order_id
7167 .and_then(|id| self.dispatch.order_identity(&id))
7168 .is_none()
7169 || !order_report_ready(&self.core, &self.dispatch, report, fills)
7170 || !fills_for_order(report, fills).any(|fill| fill.ts_event > report.ts_last)
7171 {
7172 continue;
7173 }
7174
7175 let paired = fills_for_order(report, fills).cloned().collect();
7177 let bundle = ExecutionReport::OrderWithFills(Box::new(report.clone()), paired);
7178 self.emitter.try_send_execution_report(bundle)?;
7179 bundled.insert(report.venue_order_id);
7180 }
7181
7182 orders.retain(|report| !bundled.contains(&report.venue_order_id));
7183 Ok(())
7184 }
7185
7186 fn prepare_oco_cancel_plans(&self, cmd: &CancelOrder) -> Option<Vec<CancelOrderPlan>> {
7187 let cache = self.core.cache();
7188 let order = cache.order(&cmd.client_order_id)?;
7189 if order.contingency_type() != Some(ContingencyType::Oco) || order.is_closed() {
7190 return None;
7191 }
7192
7193 let [peer_id] = order.linked_order_ids()? else {
7194 return None;
7195 };
7196
7197 if *peer_id == order.client_order_id() {
7198 return None;
7199 }
7200
7201 let peer = cache.order(peer_id)?;
7202 if peer.is_closed()
7203 || peer.instrument_id() != order.instrument_id()
7204 || peer.strategy_id() != order.strategy_id()
7205 || peer.account_id() != order.account_id()
7206 || validate_oco_links(&order, &peer, order.parent_order_id()).is_err()
7207 {
7208 return None;
7209 }
7210
7211 let original = self.prepare_cancel_order_plan(cmd).ok()?;
7212
7213 let expected_venue = order.venue_order_id().or_else(|| {
7214 self.dispatch
7215 .lookup_venue_order_id(&order.client_order_id())
7216 });
7217
7218 if original.venue_order_id != expected_venue {
7219 return None;
7220 }
7221
7222 let venue_order_id = peer.venue_order_id().or_else(|| {
7223 self.dispatch
7224 .order_identity(peer_id)
7225 .and_then(|identity| identity.venue_order_id())
7226 });
7227
7228 let venue_index = match venue_order_id {
7230 Some(venue) => venue.as_str().parse().ok()?,
7231 None => {
7232 let identity = self.dispatch.order_identity(peer_id)?;
7233 let original = self.dispatch.order_identity(&order.client_order_id())?;
7234 if !identity.shares_submission(&original) {
7235 return None;
7236 }
7237
7238 identity.client_order_index
7239 }
7240 };
7241
7242 Some(vec![
7243 original,
7244 CancelOrderPlan {
7245 client_order_id: *peer_id,
7246 strategy_id: peer.strategy_id(),
7247 instrument_id: peer.instrument_id(),
7248 venue_order_id,
7249 market_index: self.registry.market_index(&peer.instrument_id())?,
7250 venue_index,
7251 },
7252 ])
7253 }
7254
7255 fn child_probe_context(&self) -> Option<AckedOrderProbeContext> {
7256 let credential = self.credential.as_ref()?;
7257 self.fanout_dispatch_context(credential)
7258 .ok()
7259 .map(|context| context.probe_context())
7260 }
7261
7262 fn report_client(&self) -> LighterReportClient {
7263 LighterReportClient {
7264 http_client: self.http_client.clone(),
7265 credential: self.credential.clone(),
7266 account_id: self.core.account_id,
7267 registry: Arc::clone(&self.registry),
7268 clock: self.clock,
7269 markets: self.dispatch.active_markets.clone(),
7270 dispatch: self.dispatch.clone(),
7271 }
7272 }
7273
7274 fn report_instruments(
7275 &self,
7276 instrument_id: Option<InstrumentId>,
7277 ) -> AHashMap<InstrumentId, InstrumentAny> {
7278 let cache = self.core.cache();
7279 match instrument_id {
7280 Some(instrument_id) => cache
7281 .instrument(&instrument_id)
7282 .map(|instrument| (instrument_id, instrument.clone()))
7283 .into_iter()
7284 .collect(),
7285 None => cache
7286 .instruments(&self.core.venue, None)
7287 .into_iter()
7288 .map(|instrument| (instrument.id(), instrument.clone()))
7289 .collect(),
7290 }
7291 }
7292
7293 fn cached_position_reports(
7294 &self,
7295 cmd: &GeneratePositionStatusReports,
7296 ) -> anyhow::Result<(Vec<PositionStatusReport>, bool, Option<AHashSet<i64>>)> {
7297 cached_position_reports(&self.core, &self.dispatch, &self.registry, cmd)
7298 }
7299
7300 async fn paginate_fill_reports(&self, cmd: &GenerateFillReports) -> anyhow::Result<FillSweep> {
7301 let client = self.report_client();
7302 let instruments = self.report_instruments(cmd.instrument_id);
7303 let mut markets = BTreeSet::new();
7304 let result = client
7305 .collect_fill_reports(cmd, &instruments, &mut markets)
7306 .await;
7307 let result = finish_fill_reports(&self.core, &self.dispatch, markets, result);
7308
7309 if let Some(context) = self.child_probe_context() {
7310 spawn_child_recoveries(&context);
7311 }
7312
7313 result
7314 }
7315}
7316
7317struct LighterReportClient {
7318 http_client: LighterHttpClient,
7319 credential: Option<Credential>,
7320 account_id: AccountId,
7321 registry: Arc<MarketRegistry>,
7322 clock: &'static AtomicTime,
7323 markets: Arc<dashmap::DashSet<i64>>,
7324 dispatch: WsDispatchState,
7325}
7326
7327#[derive(Default)]
7328struct OrderReportSweep {
7329 orders: Vec<(i64, OrderStatusReport)>,
7330 reductions: AHashMap<VenueOrderId, Decimal>,
7331 markets: BTreeSet<i64>,
7332 error: Option<anyhow::Error>,
7333 partial: bool,
7334}
7335
7336impl LighterReportClient {
7337 async fn collect_order_status_reports(
7338 &self,
7339 cmd: &GenerateOrderStatusReports,
7340 ) -> OrderReportSweep {
7341 let mut sweep = OrderReportSweep {
7342 markets: self.markets.iter().map(|market| *market).collect(),
7343 ..Default::default()
7344 };
7345
7346 sweep.error = self.collect_orders(cmd, &mut sweep).await.err();
7347 sweep
7348 }
7349
7350 async fn collect_orders(
7351 &self,
7352 cmd: &GenerateOrderStatusReports,
7353 sweep: &mut OrderReportSweep,
7354 ) -> anyhow::Result<()> {
7355 let Some(credential) = &self.credential else {
7356 log::warn!("Lighter generate_order_status_reports: no credentials");
7357
7358 return Ok(());
7359 };
7360
7361 let auth = build_auth_token_for(credential)
7362 .context("failed to mint Lighter auth token for report fetch")?;
7363 let ts_init = self.clock.get_time_ns();
7364
7365 if cmd.instrument_id.is_none() && sweep.markets.is_empty() {
7374 collect_active_markets_from_inactive_orders(
7375 &self.http_client,
7376 &mut sweep.markets,
7377 credential,
7378 &auth,
7379 format_between_timestamps(cmd.start, cmd.end, ts_init),
7380 )
7381 .await?;
7382 }
7383
7384 sweep
7385 .markets
7386 .extend(self.markets.iter().map(|market| *market));
7387
7388 let market_indices = match cmd.instrument_id {
7389 Some(id) => match self.registry.market_index(&id) {
7390 Some(idx) => vec![idx],
7391 None => {
7392 anyhow::bail!("no Lighter market_index for order report instrument {id}",);
7393 }
7394 },
7395 None => sweep.markets.iter().copied().collect::<Vec<_>>(),
7396 };
7397
7398 if market_indices.is_empty() {
7399 log::debug!(
7400 "Lighter generate_order_status_reports: no active markets yet; returning empty",
7401 );
7402
7403 return Ok(());
7404 }
7405
7406 sweep.partial = true;
7407 self.collect_orders_active(market_indices, credential, &auth, ts_init, sweep)
7408 .await?;
7409
7410 if !cmd.open_only {
7418 self.collect_orders_inactive(cmd, credential, &auth, ts_init, sweep)
7419 .await?;
7420 }
7421
7422 Ok(())
7423 }
7424
7425 async fn collect_orders_active(
7426 &self,
7427 market_indices: Vec<i64>,
7428 credential: &Credential,
7429 auth: &SecretString,
7430 ts_init: UnixNanos,
7431 sweep: &mut OrderReportSweep,
7432 ) -> anyhow::Result<()> {
7433 let mut active_errors = Vec::new();
7434
7435 for market_index in market_indices {
7441 let query = Zeroizing::new(LighterAccountActiveOrdersQuery {
7442 authorization: None,
7443 auth: Some(auth.clone()),
7444 account_index: credential.account_index(),
7445 market_id: market_index,
7446 });
7447
7448 let active = match self.http_client.get_account_active_orders(&query).await {
7449 Ok(response) => response,
7450 Err(e) => {
7451 let detail = format!(
7452 "failed to fetch Lighter active orders for market_index={market_index}: {}",
7453 scrub_auth(&format!("{e:#}")),
7454 );
7455 log::warn!("{detail}",);
7456 active_errors.push(detail);
7457 continue;
7458 }
7459 };
7460
7461 for order in &active.orders {
7462 sweep.markets.insert(order.market_index);
7463
7464 let Some(report) = parse_http_order_to_report(
7465 order,
7466 &self.registry,
7467 self.account_id,
7468 ts_init,
7469 &self.dispatch,
7470 ) else {
7471 let detail = format!(
7472 "failed to parse Lighter active order {} for market_index={market_index}",
7473 order.order_id,
7474 );
7475 log::warn!("{detail}");
7476 active_errors.push(detail);
7477 continue;
7478 };
7479
7480 let reduced_qty = order.initial_base_amount - report.quantity.as_decimal();
7481 if reduced_qty > Decimal::ZERO {
7482 sweep.reductions.insert(report.venue_order_id, reduced_qty);
7483 }
7484
7485 sweep.orders.push((order.client_order_index, report));
7486 }
7487 }
7488
7489 if !active_errors.is_empty() {
7490 anyhow::bail!(active_errors.join("; "));
7491 }
7492
7493 Ok(())
7494 }
7495
7496 async fn collect_orders_inactive(
7497 &self,
7498 cmd: &GenerateOrderStatusReports,
7499 credential: &Credential,
7500 auth: &SecretString,
7501 ts_init: UnixNanos,
7502 sweep: &mut OrderReportSweep,
7503 ) -> anyhow::Result<()> {
7504 sweep
7505 .markets
7506 .extend(self.markets.iter().map(|market| *market));
7507
7508 let inactive_markets: Vec<i64> = match cmd.instrument_id {
7509 Some(id) => self
7510 .registry
7511 .market_index(&id)
7512 .map(|m| vec![m])
7513 .unwrap_or_default(),
7514 None => sweep.markets.iter().copied().collect::<Vec<_>>(),
7515 };
7516
7517 let between_timestamps = format_between_timestamps(cmd.start, cmd.end, ts_init);
7518
7519 for market_id in inactive_markets {
7520 let mut cursor: Option<String> = None;
7521 let mut seen_cursors = AHashSet::new();
7522 let mut pages = 0_usize;
7523 loop {
7524 pages += 1;
7525 if pages > MAX_RECONCILIATION_PAGES {
7526 anyhow::bail!(
7527 "Lighter inactive-order reconciliation exceeded {MAX_RECONCILIATION_PAGES} pages for market_index={market_id}",
7528 );
7529 }
7530
7531 let query = Zeroizing::new(LighterAccountInactiveOrdersQuery {
7532 authorization: None,
7533 auth: Some(auth.clone()),
7534 account_index: credential.account_index(),
7535 market_id: Some(market_id),
7536 ask_filter: None,
7537 between_timestamps: between_timestamps.clone(),
7538 cursor: cursor.clone(),
7539 limit: LIGHTER_REST_PAGE_SIZE,
7540 });
7541
7542 let inactive = self.http_client.get_account_inactive_orders(&query).await.map_err(|e| {
7543 anyhow::anyhow!(
7544 "failed to fetch Lighter inactive orders for market_index={market_id}: {}",
7545 scrub_auth(&format!("{e:#}")),
7546 )
7547 })?;
7548
7549 for order in &inactive.orders {
7550 let Some(report) = parse_http_order_to_report(
7551 order,
7552 &self.registry,
7553 self.account_id,
7554 ts_init,
7555 &self.dispatch,
7556 ) else {
7557 anyhow::bail!(
7558 "failed to parse Lighter inactive order {} for market_index={market_id}",
7559 order.order_id,
7560 );
7561 };
7562
7563 if cmd.start.is_some_and(|start| report.ts_last < start)
7564 || cmd.end.is_some_and(|end| report.ts_last > end)
7565 {
7566 continue;
7567 }
7568
7569 sweep.markets.insert(order.market_index);
7570 sweep.orders.push((order.client_order_index, report));
7571 }
7572
7573 let Some(next) = inactive.next_cursor else {
7574 break;
7575 };
7576
7577 if next.is_empty() {
7578 break;
7579 }
7580
7581 anyhow::ensure!(
7582 seen_cursors.insert(next.clone()),
7583 "Lighter inactive-order reconciliation repeated cursor `{next}` for market_index={market_id}",
7584 );
7585 cursor = Some(next);
7586 }
7587 }
7588
7589 Ok(())
7590 }
7591
7592 async fn collect_fill_reports(
7593 &self,
7594 cmd: &GenerateFillReports,
7595 instruments: &AHashMap<InstrumentId, InstrumentAny>,
7596 markets: &mut BTreeSet<i64>,
7597 ) -> anyhow::Result<FillSweep> {
7598 let Some(credential) = &self.credential else {
7599 log::warn!("Lighter generate_fill_reports: no credentials");
7600 return Ok(FillSweep {
7601 reports: Vec::new(),
7602 covers_window: true,
7603 withheld_instruments: AHashSet::new(),
7604 });
7605 };
7606
7607 collect_fill_reports(
7608 &self.http_client,
7609 &self.registry,
7610 credential,
7611 self.account_id,
7612 self.clock,
7613 cmd,
7614 instruments,
7615 markets,
7616 None,
7617 )
7618 .await
7619 }
7620}
7621
7622#[expect(
7623 clippy::too_many_arguments,
7624 reason = "ordinary and child-scoped recovery share authenticated fill pagination"
7625)]
7626async fn collect_fill_reports(
7627 http_client: &LighterHttpClient,
7628 registry: &Arc<MarketRegistry>,
7629 credential: &Credential,
7630 account_id: AccountId,
7631 clock: &AtomicTime,
7632 cmd: &GenerateFillReports,
7633 instruments: &AHashMap<InstrumentId, InstrumentAny>,
7634 markets: &mut BTreeSet<i64>,
7635 order_index: Option<i64>,
7636) -> anyhow::Result<FillSweep> {
7637 let market_id = match cmd.instrument_id {
7638 Some(instrument_id) => Some(registry.market_index(&instrument_id).ok_or_else(|| {
7639 anyhow::anyhow!("no Lighter market_index for fill instrument {instrument_id}",)
7640 })?),
7641 None => None,
7642 };
7643
7644 let auth = build_auth_token_for(credential)
7645 .context("failed to mint Lighter auth token for fill fetch")?;
7646
7647 let ts_init = clock.get_time_ns();
7648 let mut reports = Vec::new();
7649 let mut cursor: Option<String> = None;
7650 let mut seen_cursors = AHashSet::new();
7651 let mut seen_in_call = AHashSet::new();
7652 let mut pages = 0_usize;
7653 let mut oldest_served: Option<UnixNanos> = None;
7654 let mut covers_window = true;
7655
7656 loop {
7657 pages += 1;
7658 anyhow::ensure!(
7659 pages <= MAX_RECONCILIATION_PAGES,
7660 "Lighter fill reconciliation exceeded {MAX_RECONCILIATION_PAGES} pages",
7661 );
7662
7663 let query = Zeroizing::new(LighterTradesQuery {
7664 authorization: None,
7665 auth: Some(auth.clone()),
7666 market_id,
7667 account_index: Some(credential.account_index()),
7668 order_index,
7669 sort_by: LighterTradeSortBy::Timestamp,
7670 sort_dir: Some(LighterSortDirection::Desc),
7671 cursor: cursor.clone(),
7672 from_timestamp: None,
7675 ask_filter: None,
7676 role: None,
7677 trade_type: None,
7678 limit: LIGHTER_REST_PAGE_SIZE,
7679 aggregate: None,
7680 });
7681
7682 let response = match http_client.get_trades(&query).await {
7683 Ok(response) => response,
7684 Err(e) => {
7685 log::warn!(
7689 "Lighter get_trades failed (market_id={:?}, account_index={}, cursor={:?}): {}",
7690 query.market_id,
7691 credential.account_index(),
7692 cursor,
7693 scrub_auth(&format!("{e:#}")),
7694 );
7695 return Err(anyhow::Error::new(e).context("failed to fetch Lighter fills"));
7696 }
7697 };
7698
7699 for trade in &response.trades {
7700 let Some(report) = parse_scoped_fill_report(
7701 trade,
7702 registry,
7703 instruments,
7704 credential.account_index(),
7705 account_id,
7706 cmd,
7707 ts_init,
7708 )?
7709 else {
7710 continue;
7711 };
7712
7713 if !seen_in_call.insert(report.trade_id) {
7714 log::debug!(
7715 "Lighter duplicate trade {} ignored within HTTP fill pagination",
7716 report.trade_id,
7717 );
7718 continue;
7719 }
7720
7721 markets.insert(trade.market_id);
7722 reports.push(report);
7723 }
7724
7725 let page_oldest = response
7726 .trades
7727 .iter()
7728 .filter_map(|trade| u64::try_from(trade.timestamp).ok())
7729 .map(|timestamp_ms| UnixNanos::from(timestamp_ms.saturating_mul(1_000_000)))
7730 .min();
7731
7732 if let Some(page_oldest) = page_oldest {
7733 oldest_served = Some(oldest_served.map_or(page_oldest, |ts| ts.min(page_oldest)));
7734 }
7735
7736 let reached_start_boundary = cmd
7737 .start
7738 .is_some_and(|start| page_oldest.is_some_and(|oldest| oldest < start));
7739
7740 if reached_start_boundary {
7741 break;
7742 }
7743
7744 match response.next_cursor {
7745 Some(next) if !next.is_empty() => {
7746 anyhow::ensure!(
7747 seen_cursors.insert(next.clone()),
7748 "Lighter fill reconciliation repeated cursor `{next}`",
7749 );
7750 cursor = Some(next);
7751 }
7752 _ => {
7753 if let (Some(start), Some(oldest)) = (cmd.start, oldest_served) {
7759 covers_window = false;
7760
7761 log::warn!(
7762 "Lighter fill reports do not cover {} to {}: trade pagination exhausted before the requested start; the venue `export` endpoint serves full history",
7763 unix_nanos_to_iso8601(start),
7764 unix_nanos_to_iso8601(oldest),
7765 );
7766 }
7767
7768 break;
7769 }
7770 }
7771 }
7772
7773 Ok(FillSweep {
7774 reports,
7775 covers_window,
7776 withheld_instruments: AHashSet::new(),
7777 })
7778}
7779
7780fn parse_scoped_fill_report(
7781 trade: &LighterTrade,
7782 registry: &MarketRegistry,
7783 instruments: &AHashMap<InstrumentId, InstrumentAny>,
7784 account_index: i64,
7785 account_id: AccountId,
7786 cmd: &GenerateFillReports,
7787 ts_init: UnixNanos,
7788) -> anyhow::Result<Option<FillReport>> {
7789 let instrument_id = registry.instrument_id(trade.market_id).with_context(|| {
7790 format!(
7791 "no Lighter instrument registered for fill market_index={}",
7792 trade.market_id
7793 )
7794 })?;
7795
7796 let instrument = instruments
7797 .get(&instrument_id)
7798 .with_context(|| format!("Lighter fill instrument {instrument_id} missing from cache"))?;
7799
7800 let Some(report) = parse_ws_fill_report(trade, account_index, instrument, account_id, ts_init)
7801 .context("failed to parse Lighter fill report")?
7802 else {
7803 return Ok(None);
7804 };
7805
7806 if cmd.start.is_some_and(|start| report.ts_event < start)
7807 || cmd.end.is_some_and(|end| report.ts_event > end)
7808 {
7809 return Ok(None);
7810 }
7811
7812 Ok(Some(report))
7813}
7814
7815fn cached_position_reports(
7816 core: &ExecutionClientCore,
7817 dispatch: &WsDispatchState,
7818 registry: &Arc<MarketRegistry>,
7819 cmd: &GeneratePositionStatusReports,
7820) -> anyhow::Result<(Vec<PositionStatusReport>, bool, Option<AHashSet<i64>>)> {
7821 let (mut reports, mut coverage) = dispatch.snapshot_positions_with_coverage();
7825 let pending = execution_instruments_pending(core, dispatch);
7826 reports.retain(|report| !pending.contains(&report.instrument_id));
7827 if let Some(skipped) = &mut coverage {
7828 skipped.extend(pending.iter().filter_map(|id| registry.market_index(id)));
7829 }
7830
7831 let complete = match cmd.instrument_id {
7832 Some(instrument_id) => {
7833 let market_id = registry.market_index(&instrument_id).ok_or_else(|| {
7834 anyhow::anyhow!(
7835 "no Lighter market_index for position report instrument {instrument_id}",
7836 )
7837 })?;
7838
7839 reports.retain(|report| report.instrument_id == instrument_id);
7840 !pending.contains(&instrument_id)
7841 && coverage
7842 .as_ref()
7843 .is_some_and(|skipped| !skipped.contains(&market_id))
7844 }
7845 None => pending.is_empty() && coverage.as_ref().is_some_and(|skipped| skipped.is_empty()),
7846 };
7847
7848 Ok((reports, complete, coverage))
7849}
7850
7851fn execution_instruments_pending(
7852 core: &ExecutionClientCore,
7853 dispatch: &WsDispatchState,
7854) -> AHashSet<InstrumentId> {
7855 let events = dispatch.order_events.lock();
7856 let cache = core.cache();
7857 let mut pending = AHashSet::new();
7858 dispatch.any_order_identity(|cloid, identity, retired| {
7859 if (identity.has_inferred_fills()
7860 && cache.order(cloid).map_or(!retired, |order| {
7861 !order.is_closed() || inferred_reconciliation_trade_ids(&order).is_err()
7862 }))
7863 || dispatch.has_pending_modify(cloid)
7864 || dispatch.terminals_pending.lock().contains_key(cloid)
7865 || events.get(cloid).is_some_and(|deferred| {
7866 !deferred.fills.is_empty()
7867 || deferred
7868 .order
7869 .as_ref()
7870 .is_some_and(|order| order.status.is_terminal())
7871 || deferred.recovery_epoch.is_some()
7872 })
7873 || identity.publication_pending(cache.order(cloid).as_deref())
7874 {
7875 pending.insert(identity.instrument_id);
7876 }
7877
7878 false
7879 });
7880
7881 pending
7882}
7883
7884fn finish_order_reports(
7885 core: &ExecutionClientCore,
7886 dispatch: &WsDispatchState,
7887 sweep: OrderReportSweep,
7888 cmd: &GenerateOrderStatusReports,
7889 fills: Option<&[FillReport]>,
7890) -> anyhow::Result<Vec<OrderStatusReport>> {
7891 for market in sweep.markets {
7892 dispatch.note_active_market(market);
7893 }
7894
7895 let mut withheld = AHashSet::new();
7896 let mut reports = Vec::with_capacity(sweep.orders.len());
7897 for (client_order_index, report) in sweep.orders {
7898 let pending_cloid = core
7899 .cache()
7900 .client_order_id(&report.venue_order_id)
7901 .copied()
7902 .or_else(|| {
7903 dispatch
7904 .resolve_client_order_index_for_venue(client_order_index, report.venue_order_id)
7905 });
7906
7907 if pending_cloid
7908 .as_ref()
7909 .is_some_and(|cloid| order_shape_pending(core, dispatch, cloid))
7910 {
7911 withheld.insert(report.instrument_id);
7912 continue;
7913 }
7914
7915 restore_reconciled_order(
7916 core,
7917 dispatch,
7918 report.venue_order_id,
7919 client_order_index,
7920 report.order_status.is_closed(),
7921 );
7922
7923 if let Some(reduced_qty) = sweep.reductions.get(&report.venue_order_id) {
7924 dispatch.record_order_reduction(
7925 client_order_index,
7926 report.venue_order_id,
7927 *reduced_qty,
7928 );
7929 }
7930
7931 let report = dispatch.translate_order_cloid(report);
7932 let report = dispatch.preserve_pending_order_status(report);
7933 if report
7934 .client_order_id
7935 .as_ref()
7936 .is_some_and(|cloid| order_shape_pending(core, dispatch, cloid))
7937 {
7938 withheld.insert(report.instrument_id);
7939 continue;
7940 }
7941
7942 if !order_report_collectible(core, dispatch, &report, fills) {
7943 withheld.insert(report.instrument_id);
7944 continue;
7945 }
7946
7947 dispatch.seed_accepted_from_report(&report);
7948 reports.push(report);
7949 }
7950
7951 if let Some(error) = sweep.error {
7952 return Err(if sweep.partial {
7953 incomplete_order_reports(reports, error.to_string(), withheld)
7954 } else {
7955 error
7956 });
7957 }
7958
7959 if !withheld.is_empty() {
7960 return Err(incomplete_order_reports(
7961 reports,
7962 "Lighter execution confirmation is pending".to_string(),
7963 withheld,
7964 ));
7965 }
7966
7967 LighterExecutionClient::log_report_receipt(
7968 reports.len(),
7969 "OrderStatusReport",
7970 cmd.log_receipt_level,
7971 );
7972 Ok(reports)
7973}
7974
7975fn finish_fill_reports(
7976 core: &ExecutionClientCore,
7977 dispatch: &WsDispatchState,
7978 markets: BTreeSet<i64>,
7979 result: anyhow::Result<FillSweep>,
7980) -> anyhow::Result<FillSweep> {
7981 for market in markets {
7982 dispatch.note_active_market(market);
7983 }
7984
7985 let mut sweep = result?;
7986 let mut reports = Vec::with_capacity(sweep.reports.len());
7987
7988 for mut report in sweep.reports {
7989 defer_reconciled_fill(core, dispatch, &mut report);
7990
7991 let report = dispatch.translate_fill_cloid(report);
7994 if report.client_order_id.as_ref().is_some_and(|cloid| {
7995 order_shape_pending(core, dispatch, cloid)
7996 || (dispatch
7997 .order_identity(cloid)
7998 .is_some_and(|identity| identity.child_fill_baseline().is_some())
7999 && core
8000 .cache()
8001 .order(cloid)
8002 .is_none_or(|order| !order.is_closed()))
8003 }) {
8004 sweep.covers_window = false;
8005 sweep.withheld_instruments.insert(report.instrument_id);
8006 continue;
8007 }
8008
8009 if matches!(
8010 dispatch.trade_source(report.trade_id),
8011 Some(TradeDedupSource::Live)
8012 ) {
8013 log::debug!(
8014 "Lighter trade {} ignored in HTTP fill reports after live delivery",
8015 report.trade_id
8016 );
8017 continue;
8018 }
8019
8020 reports.push(report);
8021 }
8022
8023 sweep.reports = reports;
8024 Ok(sweep)
8025}
8026
8027fn defer_reconciled_fill(
8028 core: &ExecutionClientCore,
8029 dispatch: &WsDispatchState,
8030 report: &mut FillReport,
8031) {
8032 let Some((cloid, identity)) = restore_fill_identity(core, dispatch, report) else {
8033 return;
8034 };
8035
8036 report.client_order_id = Some(cloid);
8037
8038 if identity.has_inferred_fills() {
8039 return;
8040 }
8041
8042 let Some(fill) = child_fill_from_report(report, &identity, core.trader_id) else {
8043 return;
8044 };
8045
8046 if core
8047 .cache()
8048 .order(&cloid)
8049 .is_some_and(|order| order.is_duplicate_fill(&fill))
8050 {
8051 return;
8052 }
8053
8054 let mut events = dispatch.order_events.lock();
8055 let baseline = identity.child_fill_baseline();
8056 let new_fill = baseline
8057 .as_ref()
8058 .is_none_or(|baseline| !baseline.trade_ids.contains(&fill.trade_id));
8059 if new_fill
8060 && (baseline.is_some()
8061 || dispatch.has_pending_modify(&cloid)
8062 || position_fills_pending(&identity, &events, dispatch))
8063 {
8064 dispatch.defer_fill(events.entry(cloid).or_default(), fill);
8065 }
8066}
8067
8068fn restore_fill_identity(
8069 core: &ExecutionClientCore,
8070 dispatch: &WsDispatchState,
8071 report: &FillReport,
8072) -> Option<(ClientOrderId, OrderIdentity)> {
8073 let cached_order = {
8074 let cache = core.cache();
8075 cache
8076 .client_order_id(&report.venue_order_id)
8077 .and_then(|cloid| cache.order_owned(cloid))
8078 .filter(|order| {
8079 !order.strategy_id().is_external()
8080 && order.instrument_id() == report.instrument_id
8081 && order.account_id() == Some(report.account_id)
8082 })
8083 }?;
8084
8085 let cloid = cached_order.client_order_id();
8086 if dispatch.order_identity(&cloid).is_none() {
8087 let index = report
8088 .client_order_id
8089 .and_then(|id| id.as_str().parse::<i64>().ok())
8090 .unwrap_or(0);
8091 dispatch.seed_order_restoration(cached_order);
8092 dispatch.restore_cached_order(report.venue_order_id, index, report.instrument_id);
8093 }
8094
8095 dispatch
8096 .order_identity(&cloid)
8097 .map(|identity| (cloid, identity))
8098}
8099
8100fn ensure_order_report_ready(
8101 core: &ExecutionClientCore,
8102 dispatch: &WsDispatchState,
8103 report: &OrderStatusReport,
8104) -> anyhow::Result<()> {
8105 anyhow::ensure!(
8106 order_report_collectible(core, dispatch, report, None),
8107 "Lighter order report deferred until execution state is applied",
8108 );
8109 Ok(())
8110}
8111
8112fn order_report_collectible(
8113 core: &ExecutionClientCore,
8114 dispatch: &WsDispatchState,
8115 report: &OrderStatusReport,
8116 fills: Option<&[FillReport]>,
8117) -> bool {
8118 if fills.is_none()
8120 && matches!(
8121 report.order_status,
8122 OrderStatus::Canceled | OrderStatus::Expired
8123 )
8124 {
8125 order_report_shape_ready(core, dispatch, report)
8126 } else {
8127 order_report_ready(core, dispatch, report, fills.unwrap_or_default())
8128 }
8129}
8130
8131fn order_report_ready(
8132 core: &ExecutionClientCore,
8133 dispatch: &WsDispatchState,
8134 report: &OrderStatusReport,
8135 fills: &[FillReport],
8136) -> bool {
8137 order_report_shape_ready(core, dispatch, report)
8138 && report
8139 .client_order_id
8140 .and_then(|cloid| dispatch.order_identity(&cloid))
8141 .is_none_or(|identity| identity.report_fills_covered(report, fills))
8142}
8143
8144fn order_report_shape_ready(
8145 core: &ExecutionClientCore,
8146 dispatch: &WsDispatchState,
8147 report: &OrderStatusReport,
8148) -> bool {
8149 let Some(cloid) = report.client_order_id else {
8150 return true;
8151 };
8152
8153 let pending = order_shape_pending(core, dispatch, &cloid);
8154
8155 if let Some(identity) = dispatch.order_identity(&cloid)
8156 && identity.child_fill_baseline().is_some()
8157 && core
8158 .cache()
8159 .order(&cloid)
8160 .is_none_or(|order| report.filled_qty != order.filled_qty())
8161 {
8162 return false;
8163 }
8164
8165 if !pending {
8166 return true;
8167 }
8168
8169 report.order_status == OrderStatus::PendingUpdate
8170 && core.cache().order(&cloid).is_some_and(|order| {
8171 order.quantity() == report.quantity
8172 && order.price() == report.price
8173 && order.trigger_price() == report.trigger_price
8174 && order.filled_qty() == report.filled_qty
8175 })
8176}
8177
8178fn order_shape_pending(
8179 core: &ExecutionClientCore,
8180 dispatch: &WsDispatchState,
8181 cloid: &ClientOrderId,
8182) -> bool {
8183 let events = dispatch.order_events.lock();
8184 if dispatch.has_pending_modify(cloid) {
8185 return true;
8186 }
8187
8188 let Some(identity) = dispatch.order_identity(cloid) else {
8189 return false;
8190 };
8191
8192 if let Some(order) = core.cache().order(cloid) {
8193 identity.sync_fills(&order);
8194 }
8195
8196 if identity.has_inferred_fills() {
8197 return true;
8198 }
8199
8200 if dispatch.terminals_pending.lock().contains_key(cloid) {
8201 return true;
8202 }
8203
8204 if position_fills_pending(&identity, &events, dispatch) {
8205 return true;
8206 }
8207
8208 let cache = core.cache();
8209
8210 if dispatch.any_order_identity(|id, other, _| {
8211 other.strategy_id == identity.strategy_id
8212 && other.instrument_id == identity.instrument_id
8213 && other.publication_pending(cache.order(id).as_deref())
8214 }) {
8215 return true;
8216 }
8217
8218 let order = cache.order(cloid);
8219 let pending = identity.publication_pending(order.as_deref());
8220 if !pending && order.as_ref().is_some_and(|order| order.is_closed()) {
8221 dispatch.retire_order_identity(cloid);
8222 }
8223
8224 pending
8225}
8226
8227fn restore_reconciled_order(
8228 core: &ExecutionClientCore,
8229 dispatch: &WsDispatchState,
8230 venue_order_id: VenueOrderId,
8231 client_order_index: i64,
8232 terminal: bool,
8233) {
8234 let cached_order = {
8235 let cache = core.cache();
8236
8237 let Some(cloid) = cache.client_order_id(&venue_order_id).copied() else {
8238 return;
8239 };
8240
8241 let Some(order) = cache.order_owned(&cloid) else {
8242 log::warn!(
8243 "Ignoring reconciled Lighter order missing from cache: cloid={cloid}, venue_order_id={venue_order_id}",
8244 );
8245 return;
8246 };
8247
8248 order
8249 };
8250
8251 if cached_order.instrument_id().venue != core.venue
8252 || cached_order.account_id() != Some(core.account_id)
8253 {
8254 log::warn!(
8255 "Ignoring reconciled Lighter order outside this client: cloid={}, venue_order_id={venue_order_id}",
8256 cached_order.client_order_id(),
8257 );
8258 return;
8259 }
8260
8261 if order_shape_pending(core, dispatch, &cached_order.client_order_id()) {
8262 return;
8263 }
8264
8265 let terminal =
8266 terminal && (cached_order.parent_order_id().is_none() || cached_order.is_closed());
8267
8268 if let Err(e) = dispatch.restore_reconciled_order(
8269 &cached_order,
8270 client_order_index,
8271 venue_order_id,
8272 terminal,
8273 ) {
8274 log::warn!(
8275 "Ignoring conflicting Lighter reconciliation identity: cloid={}, venue_order_id={venue_order_id}, client_order_index={}, error={e}",
8276 cached_order.client_order_id(),
8277 client_order_index,
8278 );
8279 }
8280}
8281
8282fn grouped_submit_denial_reason(error: &anyhow::Error) -> String {
8283 match error.downcast_ref::<OrderDeniedReason>() {
8284 Some(reason) => reason.to_string(),
8285 None => OrderDeniedReason::UnsupportedOrderList {
8286 detail: format!("Lighter grouped submit failed: {error}"),
8287 }
8288 .to_string(),
8289 }
8290}
8291
8292fn local_submit_denial_reason(
8293 order: &OrderAny,
8294 instrument: Option<&InstrumentAny>,
8295) -> Option<anyhow::Error> {
8296 if instrument.is_none() {
8297 return Some(anyhow::anyhow!(
8298 OrderDeniedReason::InstrumentNotFound {
8299 instrument_id: order.instrument_id(),
8300 }
8301 .to_string(),
8302 ));
8303 }
8304
8305 if !is_lighter_supported_order_type(order.order_type()) {
8306 return Some(anyhow::anyhow!(unsupported_lighter_order_type_reason(
8307 order.order_type()
8308 )));
8309 }
8310
8311 if is_lighter_limit_style_order(order.order_type()) && order.price().is_none() {
8312 return Some(anyhow::anyhow!(
8313 OrderDeniedReason::ValidationFailed {
8314 detail: "Lighter limit-style orders require a limit price".to_string(),
8315 }
8316 .to_string(),
8317 ));
8318 }
8319
8320 if order.is_quote_quantity() {
8321 return Some(anyhow::anyhow!(
8322 OrderDeniedReason::unsupported_quote_quantity()
8323 ));
8324 }
8325
8326 if order.display_qty().is_some() {
8327 return Some(anyhow::anyhow!(
8328 OrderDeniedReason::ValidationFailed {
8329 detail: "Lighter orders do not support display_qty iceberg instructions"
8330 .to_string(),
8331 }
8332 .to_string(),
8333 ));
8334 }
8335
8336 if is_lighter_spot_order(order, instrument) && is_lighter_conditional_order(order.order_type())
8337 {
8338 let denied = OrderDeniedReason::UnsupportedOrderType {
8339 order_type: order.order_type(),
8340 };
8341
8342 return Some(anyhow::anyhow!(
8343 "{denied}; Lighter spot markets do not support conditional orders",
8344 ));
8345 }
8346
8347 nautilus_to_lighter_tif(
8348 order.order_type(),
8349 order.time_in_force(),
8350 order.is_post_only(),
8351 )
8352 .err()
8353 .map(|e| {
8354 let denied = OrderDeniedReason::UnsupportedTimeInForce(order.time_in_force());
8355 anyhow::anyhow!("{denied}; {e}")
8356 })
8357}
8358
8359fn unsupported_lighter_order_type_reason(order_type: OrderType) -> String {
8360 let denied = OrderDeniedReason::UnsupportedOrderType { order_type };
8361 format!(
8362 "{denied}; Lighter supports MARKET, LIMIT, STOP_MARKET, STOP_LIMIT, MARKET_IF_TOUCHED, and LIMIT_IF_TOUCHED",
8363 )
8364}
8365
8366fn is_grouped_order(order: &OrderAny) -> bool {
8367 order.contingency_type().is_some() || order.parent_order_id().is_some()
8368}
8369
8370fn grouped_order_indices(orders: &[OrderAny]) -> anyhow::Result<(LighterGroupingType, Vec<usize>)> {
8371 anyhow::ensure!(
8372 (2..=3).contains(&orders.len()),
8373 "Lighter groups require two or three orders"
8374 );
8375 let first = &orders[0];
8376 anyhow::ensure!(
8377 orders
8378 .iter()
8379 .enumerate()
8380 .all(|(index, order)| orders[..index]
8381 .iter()
8382 .all(|other| other.client_order_id() != order.client_order_id())),
8383 "Lighter groups require unique client order IDs"
8384 );
8385 anyhow::ensure!(
8386 orders.iter().all(|order| !order.is_closed()
8387 && order.instrument_id() == first.instrument_id()
8388 && order.strategy_id() == first.strategy_id()
8389 && order.order_list_id() == first.order_list_id()),
8390 "Lighter groups require open orders from one instrument, strategy, and order list"
8391 );
8392 anyhow::ensure!(
8393 !orders
8394 .iter()
8395 .any(|order| order.contingency_type() == Some(ContingencyType::Ouo)),
8396 "Lighter native groups do not support Nautilus OUO quantity updates"
8397 );
8398
8399 let parents: Vec<_> = orders
8400 .iter()
8401 .enumerate()
8402 .filter(|(_, order)| {
8403 order.contingency_type() == Some(ContingencyType::Oto)
8404 && order.parent_order_id().is_none()
8405 })
8406 .map(|(index, _)| index)
8407 .collect();
8408
8409 if parents.is_empty() {
8410 anyhow::ensure!(
8411 orders.len() == 2,
8412 "Lighter OCO requires exactly two siblings"
8413 );
8414 validate_oco_links(&orders[0], &orders[1], None)?;
8415 return Ok((LighterGroupingType::OneCancelsTheOther, vec![0, 1]));
8416 }
8417
8418 anyhow::ensure!(
8419 parents.len() == 1,
8420 "Lighter OTO groups require exactly one parent"
8421 );
8422 let parent_index = parents[0];
8423 let parent = &orders[parent_index];
8424 anyhow::ensure!(
8425 matches!(parent.order_type(), OrderType::Market | OrderType::Limit),
8426 "Lighter OTO parents must be MARKET or LIMIT orders"
8427 );
8428 let children: Vec<_> = orders
8429 .iter()
8430 .enumerate()
8431 .filter(|(index, _)| *index != parent_index)
8432 .collect();
8433 let child_ids: Vec<_> = children
8434 .iter()
8435 .map(|(_, order)| order.client_order_id())
8436 .collect();
8437 let links = parent
8438 .linked_order_ids()
8439 .context("Lighter OTO parent requires linked children")?;
8440 anyhow::ensure!(
8441 links.len() == child_ids.len() && child_ids.iter().all(|id| links.contains(id)),
8442 "Lighter OTO parent links must identify every child exactly once"
8443 );
8444 anyhow::ensure!(
8445 children
8446 .iter()
8447 .all(|(_, order)| order.parent_order_id() == Some(parent.client_order_id())),
8448 "Lighter OTO children must reference their parent"
8449 );
8450
8451 let grouping_type = if children.len() == 1 {
8452 anyhow::ensure!(
8453 children[0].1.contingency_type().is_none()
8454 && children[0].1.linked_order_ids().is_none(),
8455 "Lighter OTO child must not have a sibling contingency"
8456 );
8457 LighterGroupingType::OneTriggersTheOther
8458 } else {
8459 validate_oco_links(children[0].1, children[1].1, Some(parent.client_order_id()))?;
8460 LighterGroupingType::OneTriggersOneCancelsTheOther
8461 };
8462
8463 Ok((
8464 grouping_type,
8465 std::iter::once(parent_index)
8466 .chain(children.iter().map(|(index, _)| *index))
8467 .collect(),
8468 ))
8469}
8470
8471fn validate_oco_links(
8472 first: &OrderAny,
8473 second: &OrderAny,
8474 parent: Option<ClientOrderId>,
8475) -> anyhow::Result<()> {
8476 let orders = [first, second];
8477 for (index, order) in orders.iter().enumerate() {
8478 anyhow::ensure!(
8479 order.contingency_type() == Some(ContingencyType::Oco)
8480 && order.parent_order_id() == parent,
8481 "Lighter OCO siblings must have matching OCO relationships"
8482 );
8483 anyhow::ensure!(
8484 order
8485 .linked_order_ids()
8486 .is_some_and(|links| links == [orders[1 - index].client_order_id()]),
8487 "Lighter OCO siblings must link only to each other"
8488 );
8489 }
8490
8491 Ok(())
8492}
8493
8494fn prepare_group_bodies(
8495 grouping_type: LighterGroupingType,
8496 plans: &mut [CreateOrderPlan],
8497 use_gtd: bool,
8498) -> anyhow::Result<()> {
8499 let has_parent = grouping_type != LighterGroupingType::OneCancelsTheOther;
8500 let parent_quantity = plans[0].order.quantity();
8501 let child_start = usize::from(has_parent);
8502
8503 let default_expiry = plans
8504 .iter()
8505 .find(|plan| {
8506 plan.order_expiry != 0 && !(plan.order.time_in_force() == TimeInForce::Gtd && use_gtd)
8507 })
8508 .map_or(plans[child_start].order_expiry, |plan| plan.order_expiry);
8509
8510 for plan in plans.iter_mut() {
8511 if plan.order_expiry != 0 && !(plan.order.time_in_force() == TimeInForce::Gtd && use_gtd) {
8512 plan.order_expiry = default_expiry;
8513 }
8514 }
8515
8516 let expiry = plans[child_start].order_expiry;
8517 let parent_side = plans[0].order.order_side();
8518 for child in &mut plans[child_start..] {
8519 anyhow::ensure!(
8520 is_lighter_conditional_order(child.order.order_type()) && child.order.is_reduce_only(),
8521 "Lighter group children require reduce-only SL/TP orders"
8522 );
8523 anyhow::ensure!(
8524 child.order.quantity() == parent_quantity,
8525 "Lighter group quantities must match before parent-fill sizing"
8526 );
8527 anyhow::ensure!(
8528 (child.order.order_side() != parent_side) == has_parent,
8529 "Lighter OTO children must oppose their parent; OCO siblings must have the same side"
8530 );
8531 anyhow::ensure!(
8532 child.order_expiry == expiry && expiry != 0,
8533 "Lighter group children require matching nonzero expiries"
8534 );
8535
8536 if has_parent {
8537 child.base_amount = 0;
8538 }
8539 }
8540
8541 anyhow::ensure!(
8542 plans[0].order_expiry == 0 || plans[0].order_expiry == expiry,
8543 "Lighter parent and child expiries must match"
8544 );
8545
8546 if plans.len() - child_start == 2 {
8547 let stop_count = plans[child_start..]
8548 .iter()
8549 .filter(|plan| {
8550 matches!(
8551 plan.order.order_type(),
8552 OrderType::StopMarket | OrderType::StopLimit
8553 )
8554 })
8555 .count();
8556
8557 anyhow::ensure!(
8558 stop_count == 1,
8559 "Lighter OCO requires one stop-loss and one take-profit"
8560 );
8561 }
8562
8563 Ok(())
8564}
8565
8566fn is_lighter_spot_order(order: &OrderAny, instrument: Option<&InstrumentAny>) -> bool {
8567 instrument.is_some_and(|instrument| matches!(instrument, InstrumentAny::CurrencyPair(_)))
8568 || product_type_from_instrument_id(&order.instrument_id()) == Some(LighterProductType::Spot)
8569}
8570
8571fn is_lighter_supported_order_type(order_type: OrderType) -> bool {
8572 matches!(
8573 order_type,
8574 OrderType::Market
8575 | OrderType::Limit
8576 | OrderType::StopMarket
8577 | OrderType::StopLimit
8578 | OrderType::MarketIfTouched
8579 | OrderType::LimitIfTouched
8580 )
8581}
8582
8583fn is_lighter_limit_style_order(order_type: OrderType) -> bool {
8584 matches!(
8585 order_type,
8586 OrderType::Limit | OrderType::StopLimit | OrderType::LimitIfTouched
8587 )
8588}
8589
8590fn is_lighter_conditional_order(order_type: OrderType) -> bool {
8591 matches!(
8592 order_type,
8593 OrderType::StopMarket
8594 | OrderType::StopLimit
8595 | OrderType::MarketIfTouched
8596 | OrderType::LimitIfTouched
8597 )
8598}
8599
8600async fn collect_active_markets_from_inactive_orders(
8601 http_client: &LighterHttpClient,
8602 markets: &mut BTreeSet<i64>,
8603 credential: &Credential,
8604 auth: &SecretString,
8605 between_timestamps: Option<String>,
8606) -> anyhow::Result<()> {
8607 let mut cursor: Option<String> = None;
8608 let mut seen_cursors = AHashSet::new();
8609 let mut orders_seen = 0_usize;
8610 let mut pages = 0_usize;
8611 loop {
8612 pages += 1;
8613 anyhow::ensure!(
8614 pages <= MAX_RECONCILIATION_PAGES,
8615 "Lighter active-market seed exceeded {MAX_RECONCILIATION_PAGES} pages",
8616 );
8617
8618 let query = Zeroizing::new(LighterAccountInactiveOrdersQuery {
8619 authorization: None,
8620 auth: Some(auth.clone()),
8621 account_index: credential.account_index(),
8622 market_id: None,
8623 ask_filter: None,
8624 between_timestamps: between_timestamps.clone(),
8625 cursor: cursor.clone(),
8626 limit: LIGHTER_REST_PAGE_SIZE,
8627 });
8628 let response = http_client
8629 .get_account_inactive_orders(&query)
8630 .await
8631 .context("failed to seed Lighter active markets from inactive orders")?;
8632
8633 for order in &response.orders {
8634 markets.insert(order.market_index);
8635 orders_seen += 1;
8636 }
8637
8638 match response.next_cursor {
8639 Some(next) if !next.is_empty() => {
8640 anyhow::ensure!(
8641 seen_cursors.insert(next.clone()),
8642 "Lighter active-market seed repeated cursor `{next}`",
8643 );
8644 cursor = Some(next);
8645 }
8646 _ => break,
8647 }
8648 }
8649
8650 if orders_seen > 0 {
8651 log::debug!("Seeded Lighter active markets from {orders_seen} inactive order report(s)");
8652 }
8653
8654 Ok(())
8655}
8656
8657fn cancel_order_from_cancel_all(
8658 cmd: &CancelAllOrders,
8659 client_order_id: ClientOrderId,
8660 strategy_id: StrategyId,
8661) -> CancelOrder {
8662 CancelOrder {
8663 trader_id: cmd.trader_id,
8664 client_id: cmd.client_id,
8665 strategy_id,
8666 instrument_id: cmd.instrument_id,
8667 client_order_id,
8668 venue_order_id: None,
8669 command_id: cmd.command_id,
8670 ts_init: cmd.ts_init,
8671 params: cmd.params.clone(),
8672 correlation_id: cmd.correlation_id,
8673 causation_id: cmd.causation_id,
8674 }
8675}
8676
8677fn validate_order_amount(
8678 instrument: &InstrumentAny,
8679 quantity: Quantity,
8680 price_ticks: u32,
8681 price_precision: u8,
8682) -> anyhow::Result<()> {
8683 if let Some(min_quantity) = instrument.min_quantity() {
8684 anyhow::ensure!(
8685 quantity >= min_quantity,
8686 "quantity `{quantity}` below Lighter min_base_amount `{min_quantity}` for {}",
8687 instrument.id(),
8688 );
8689 }
8690
8691 if let Some(min_notional) = instrument.min_notional() {
8692 let price = decimal_from_ticks(price_ticks, price_precision);
8693 let notional = quantity.as_decimal() * price;
8694 anyhow::ensure!(
8695 notional >= min_notional.as_decimal(),
8696 "order notional `{notional}` below Lighter min_quote_amount `{}` for {}",
8697 min_notional.as_decimal(),
8698 instrument.id(),
8699 );
8700 }
8701
8702 Ok(())
8703}
8704
8705fn decimal_from_ticks(ticks: u32, decimals: u8) -> Decimal {
8706 Decimal::from(ticks) / Decimal::from(10_i64.pow(u32::from(decimals)))
8707}
8708
8709fn dispatch_lighter_order(
8713 order: &crate::http::models::LighterOrder,
8714 dispatch: &WsDispatchState,
8715 emitter: &ExecutionEventEmitter,
8716 registry: &Arc<MarketRegistry>,
8717 account_id: AccountId,
8718 trader_id: TraderId,
8719 ts_init: UnixNanos,
8720) {
8721 let mut events = dispatch.order_events.lock();
8722
8723 let instrument_id = match registry.instrument_id(order.market_index) {
8724 Some(id) => id,
8725 None => {
8726 log::debug!(
8727 "Lighter order frame dropped: no instrument for market_index={}",
8728 order.market_index,
8729 );
8730 return;
8731 }
8732 };
8733
8734 dispatch.note_active_market(order.market_index);
8735
8736 let instrument = match LIGHTER_INSTRUMENT_CACHE.get(&instrument_id) {
8737 Some(inst) => inst.value().clone(),
8738 None => {
8739 log::debug!("Lighter order frame dropped: instrument {instrument_id} not in cache",);
8740 return;
8741 }
8742 };
8743
8744 let venue_order_id = VenueOrderId::new(order.order_id.as_str());
8745 if !dispatch.restore_cached_order(venue_order_id, order.client_order_index, instrument_id) {
8746 return;
8747 }
8748
8749 let resolved_cloid =
8750 dispatch.resolve_live_order_cloid(order.client_order_id.as_str(), venue_order_id);
8751
8752 let identity = resolved_cloid.and_then(|cid| {
8753 dispatch
8754 .order_identities
8755 .get(&cid)
8756 .map(|entry| (cid, entry.value().clone()))
8757 });
8758
8759 let Some((cloid, identity)) = identity else {
8760 dispatch_unowned_order(order, &instrument, dispatch, emitter, account_id, ts_init);
8761 return;
8762 };
8763
8764 if identity.instrument_id != instrument_id {
8765 return;
8766 }
8767
8768 if defer_owned_order(order, &identity, cloid, &mut events, dispatch, emitter) {
8769 return;
8770 }
8771
8772 dispatch_owned_order(
8773 order,
8774 &instrument,
8775 &identity,
8776 cloid,
8777 venue_order_id,
8778 dispatch,
8779 emitter,
8780 account_id,
8781 trader_id,
8782 ts_init,
8783 );
8784}
8785
8786fn defer_owned_order(
8787 order: &LighterOrder,
8788 identity: &OrderIdentity,
8789 cloid: ClientOrderId,
8790 events: &mut AHashMap<ClientOrderId, DeferredOrderEvents>,
8791 dispatch: &WsDispatchState,
8792 emitter: &ExecutionEventEmitter,
8793) -> bool {
8794 if dispatch.has_pending_modify(&cloid) {
8795 events.entry(cloid).or_default().store_order(order);
8796 return true;
8797 }
8798
8799 if identity.child_fill_baseline().is_some() && !identity.has_inferred_fills() {
8800 if identity.has_child_publication() {
8801 return true;
8802 }
8803
8804 let deferred = events.entry(cloid).or_default();
8805
8806 let had_terminal = deferred
8807 .order
8808 .as_ref()
8809 .is_some_and(|prior| prior.status.is_terminal());
8810
8811 deferred.store_order(order);
8812 let terminal = order.status.is_terminal();
8813
8814 if terminal || had_terminal {
8815 publish_child_recovery(cloid, events, dispatch, emitter);
8816 return true;
8817 }
8818 }
8819
8820 if position_fills_pending(identity, events, dispatch) && order.status.is_terminal() {
8821 events.entry(cloid).or_default().store_order(order);
8822 return true;
8823 }
8824
8825 false
8826}
8827
8828fn dispatch_unowned_order(
8829 order: &LighterOrder,
8830 instrument: &InstrumentAny,
8831 dispatch: &WsDispatchState,
8832 emitter: &ExecutionEventEmitter,
8833 account_id: AccountId,
8834 ts_init: UnixNanos,
8835) {
8836 match parse_ws_order_status_report(order, instrument, account_id, ts_init) {
8837 Ok(mut report) => {
8838 report = dispatch.apply_order_contingencies(order, report);
8839 report = dispatch.translate_order_cloid(report);
8840 report = dispatch.preserve_pending_order_status(report);
8841 if report
8842 .client_order_id
8843 .and_then(|id| dispatch.order_identity(&id))
8844 .is_some_and(|identity| {
8845 !identity.report_fills_covered(&report, &[]) || identity.has_publication()
8846 })
8847 {
8848 return;
8849 }
8850
8851 if let Some(cloid) = &report.client_order_id {
8852 dispatch.venue_id_map.insert(*cloid, report.venue_order_id);
8853 }
8854
8855 if report.order_status.is_closed() {
8856 evict_terminal_mappings(&report, &dispatch.venue_id_map);
8857 }
8858
8859 log::debug!(
8860 "Lighter OrderStatusReport: voi={} status={:?} cloid={:?}",
8861 report.venue_order_id,
8862 report.order_status,
8863 report.client_order_id,
8864 );
8865 emitter.send_order_status_report(report);
8866 }
8867 Err(e) => {
8868 log::error!(
8869 "Failed to parse Lighter order status report: error={e}, order_id={}",
8870 order.order_id,
8871 );
8872 }
8873 }
8874}
8875
8876#[expect(
8877 clippy::too_many_arguments,
8878 reason = "tracked order publication uses the same identity and emitter context as stream dispatch"
8879)]
8880fn dispatch_owned_order(
8881 order: &crate::http::models::LighterOrder,
8882 instrument: &InstrumentAny,
8883 identity: &OrderIdentity,
8884 cloid: ClientOrderId,
8885 venue_order_id: VenueOrderId,
8886 dispatch: &WsDispatchState,
8887 emitter: &ExecutionEventEmitter,
8888 account_id: AccountId,
8889 trader_id: TraderId,
8890 ts_init: UnixNanos,
8891) {
8892 dispatch.venue_id_map.insert(cloid, venue_order_id);
8893
8894 let is_live_status = matches!(
8899 order.status,
8900 crate::common::enums::LighterOrderStatus::Pending
8901 | crate::common::enums::LighterOrderStatus::Open,
8902 );
8903 let pending_action = dispatch.pending_order_action(&cloid);
8904 if is_live_status && pending_action == Some(PendingOrderAction::Cancel) {
8905 log::debug!(
8906 "Deferring Lighter {:?} frame while cancel is pending for {cloid}",
8907 order.status,
8908 );
8909 return;
8910 }
8911
8912 let current_shape = match lighter_order_shape(order, instrument, identity.order_type) {
8913 Ok(shape) => shape,
8914 Err(e) => {
8915 log::error!(
8916 "Failed to compute Lighter order shape: error={e}, voi={venue_order_id}, cloid={cloid}",
8917 );
8918 return;
8919 }
8920 };
8921
8922 let prior_shape = dispatch.snapshot_for(&cloid);
8923 identity.record_reduction(order.initial_base_amount - current_shape.quantity.as_decimal());
8924 let shape_changed = prior_shape.map_or_else(
8925 || current_shape.quantity.as_decimal() < order.initial_base_amount,
8926 |previous| {
8927 identity.normalize_shape(previous) != identity.normalize_shape(current_shape.clone())
8928 },
8929 );
8930
8931 if shape_changed
8932 && !dispatch.accepted_was_emitted(&cloid)
8933 && (is_live_status || order.status == crate::common::enums::LighterOrderStatus::Filled)
8934 {
8935 ensure_accepted_emitted(
8936 cloid,
8937 venue_order_id,
8938 identity,
8939 account_id,
8940 trader_id,
8941 emitter,
8942 dispatch,
8943 ts_init,
8944 );
8945 }
8946
8947 let fresh_trigger = is_order_triggered(order)
8948 && TRIGGERABLE_ORDER_TYPES.contains(&identity.order_type)
8949 && !dispatch.triggered_was_emitted(&cloid);
8950
8951 if pending_action == Some(PendingOrderAction::Modify) && fresh_trigger && !shape_changed {
8952 log::debug!("Deferring Lighter trigger while modify confirmation is pending for {cloid}",);
8953 return;
8954 }
8955
8956 let open_ctx = OpenFrameContext {
8957 accepted_already_emitted: dispatch.accepted_was_emitted(&cloid),
8958 triggered_already_emitted: dispatch.triggered_was_emitted(&cloid),
8959 shape_changed,
8960 };
8961
8962 let event_opt = match parse_lighter_order_event(
8963 order, instrument, identity, cloid, account_id, trader_id, open_ctx, ts_init,
8964 ) {
8965 Ok(event) => event,
8966 Err(e) => {
8967 log::error!(
8968 "Failed to parse Lighter order event: error={e}, voi={venue_order_id}, cloid={cloid}"
8969 );
8970 return;
8971 }
8972 };
8973
8974 dispatch.store_snapshot(cloid, current_shape);
8975 if let Some(event) = event_opt {
8976 let Some(event) = defer_terminal(order, event, identity, cloid, dispatch) else {
8977 return;
8978 };
8979
8980 dispatch_tracked_order_event(
8981 event,
8982 cloid,
8983 venue_order_id,
8984 identity,
8985 account_id,
8986 trader_id,
8987 emitter,
8988 dispatch,
8989 ts_init,
8990 );
8991 }
8992
8993 if order.status == crate::common::enums::LighterOrderStatus::Filled {
8994 dispatch.clear_pending_order_action(&cloid);
8995 dispatch.retire_order_identity(&cloid);
8996 }
8997}
8998
8999#[expect(
9003 clippy::too_many_arguments,
9004 reason = "consumption-loop dispatch threads identity and emitter context"
9005)]
9006fn dispatch_lighter_trade(
9007 trade: &crate::http::models::LighterTrade,
9008 dispatch: &WsDispatchState,
9009 emitter: &ExecutionEventEmitter,
9010 registry: &Arc<MarketRegistry>,
9011 account_id: AccountId,
9012 trader_id: TraderId,
9013 account_index: Option<i64>,
9014 ts_init: UnixNanos,
9015) {
9016 let mut events = dispatch.order_events.lock();
9017
9018 let Some(account_index) = account_index else {
9019 log::debug!("Lighter trade frame dropped: no credential / account_index available",);
9020 return;
9021 };
9022
9023 let Some(instrument_id) = registry.instrument_id(trade.market_id) else {
9024 log::debug!(
9025 "Lighter trade frame dropped: no instrument for market_id={}",
9026 trade.market_id,
9027 );
9028 return;
9029 };
9030
9031 dispatch.note_active_market(trade.market_id);
9032
9033 let Some(instrument) = LIGHTER_INSTRUMENT_CACHE
9034 .get(&instrument_id)
9035 .map(|instrument| instrument.value().clone())
9036 else {
9037 log::debug!("Lighter trade frame dropped: instrument {instrument_id} not in cache",);
9038 return;
9039 };
9040
9041 let user_is_bidder = trade.bid_account_id == account_index;
9042 let user_is_asker = trade.ask_account_id == account_index;
9043 if !user_is_bidder && !user_is_asker {
9044 return;
9047 }
9048
9049 let trade_id = match parse_lighter_trade_id(trade) {
9052 Ok(id) => id,
9053 Err(e) => {
9054 log::error!("Lighter trade has invalid trade_id: {e}");
9055 return;
9056 }
9057 };
9058
9059 let (raw_client_id, client_order_index, venue_order_id) = if user_is_bidder {
9060 (
9061 trade
9062 .bid_client_id_str
9063 .as_deref()
9064 .map_or_else(|| trade.bid_client_id.to_string(), str::to_string),
9065 trade.bid_client_id,
9066 trade.bid_id_str.as_deref().map_or_else(
9067 || VenueOrderId::new(trade.bid_id.to_string()),
9068 VenueOrderId::new,
9069 ),
9070 )
9071 } else {
9072 (
9073 trade
9074 .ask_client_id_str
9075 .as_deref()
9076 .map_or_else(|| trade.ask_client_id.to_string(), str::to_string),
9077 trade.ask_client_id,
9078 trade.ask_id_str.as_deref().map_or_else(
9079 || VenueOrderId::new(trade.ask_id.to_string()),
9080 VenueOrderId::new,
9081 ),
9082 )
9083 };
9084
9085 if !dispatch.restore_cached_order(venue_order_id, client_order_index, instrument_id) {
9086 return;
9087 }
9088
9089 let resolved_cloid = dispatch.resolve_live_trade_cloid(raw_client_id.as_str(), venue_order_id);
9090 let identity = resolved_cloid.and_then(|cid| {
9091 dispatch
9092 .order_identity(&cid)
9093 .map(|identity| (cid, identity))
9094 });
9095
9096 let Some((cloid, identity)) = identity else {
9097 match parse_ws_fill_report(trade, account_index, &instrument, account_id, ts_init) {
9098 Ok(Some(mut report)) => {
9099 report = dispatch.translate_fill_cloid(report);
9100 log::debug!(
9101 "Lighter FillReport: voi={} qty={} px={} liq={:?} cloid={:?}",
9102 report.venue_order_id,
9103 report.last_qty,
9104 report.last_px,
9105 report.liquidity_side,
9106 report.client_order_id,
9107 );
9108
9109 if dispatch.mark_trade_seen(trade_id) {
9110 emitter.send_fill_report(report);
9111 }
9112 }
9113 Ok(None) => {}
9114 Err(e) => {
9115 log::error!("Failed to parse Lighter fill report: error={e}, trade_id={trade_id}",);
9116 }
9117 }
9118
9119 return;
9120 };
9121
9122 if identity.instrument_id != instrument_id {
9123 return;
9124 }
9125
9126 dispatch_owned_trade(
9127 trade,
9128 &instrument,
9129 &identity,
9130 cloid,
9131 venue_order_id,
9132 &mut events,
9133 dispatch,
9134 emitter,
9135 account_id,
9136 trader_id,
9137 account_index,
9138 trade_id,
9139 ts_init,
9140 );
9141}
9142
9143#[expect(
9144 clippy::too_many_arguments,
9145 reason = "tracked trade publication uses the stream identity and emitter context"
9146)]
9147fn dispatch_owned_trade(
9148 trade: &crate::http::models::LighterTrade,
9149 instrument: &InstrumentAny,
9150 identity: &OrderIdentity,
9151 cloid: ClientOrderId,
9152 venue_order_id: VenueOrderId,
9153 events: &mut AHashMap<ClientOrderId, DeferredOrderEvents>,
9154 dispatch: &WsDispatchState,
9155 emitter: &ExecutionEventEmitter,
9156 account_id: AccountId,
9157 trader_id: TraderId,
9158 account_index: i64,
9159 trade_id: TradeId,
9160 ts_init: UnixNanos,
9161) {
9162 if identity.has_inferred_fills() {
9163 log::warn!(
9164 "Withholding Lighter trade {trade_id} for {cloid}: inferred fills require reconciliation"
9165 );
9166 return;
9167 }
9168
9169 match parse_lighter_order_filled(
9170 trade,
9171 instrument,
9172 identity,
9173 cloid,
9174 account_id,
9175 trader_id,
9176 account_index,
9177 ts_init,
9178 ) {
9179 Ok(Some(filled)) => {
9180 ensure_accepted_emitted(
9181 cloid,
9182 venue_order_id,
9183 identity,
9184 account_id,
9185 trader_id,
9186 emitter,
9187 dispatch,
9188 ts_init,
9189 );
9190
9191 log::debug!(
9192 "Lighter OrderFilled: voi={} qty={} px={} liq={:?} cloid={cloid}",
9193 filled.venue_order_id,
9194 filled.last_qty,
9195 filled.last_px,
9196 filled.liquidity_side,
9197 );
9198
9199 if let Some(baseline) = identity.child_fill_baseline() {
9200 if baseline.trade_ids.contains(&trade_id) {
9201 return;
9202 }
9203
9204 dispatch.defer_fill(events.entry(cloid).or_default(), filled);
9205 publish_child_recovery(cloid, events, dispatch, emitter);
9206 } else if dispatch.has_pending_modify(&cloid)
9207 || position_fills_pending(identity, events, dispatch)
9208 {
9209 dispatch.defer_fill(events.entry(cloid).or_default(), filled);
9210 } else if dispatch.mark_trade_seen(trade_id) {
9211 dispatch_lighter_fill(filled, emitter, dispatch, events);
9212 }
9213 }
9214 Ok(None) => {}
9215 Err(e) => {
9216 log::error!("Failed to parse Lighter typed fill: error={e}, trade_id={trade_id}",);
9217 }
9218 }
9219}
9220
9221fn spawn_child_recoveries(context: &AckedOrderProbeContext) {
9222 let candidates = context
9223 .dispatch
9224 .order_events
9225 .lock()
9226 .iter()
9227 .filter_map(|(cloid, deferred)| (!deferred.fills.is_empty()).then_some(*cloid))
9228 .collect::<Vec<_>>();
9229
9230 for cloid in candidates {
9231 spawn_child_recovery(cloid, context);
9232 }
9233}
9234
9235fn request_child_report_recoveries(
9236 core: &ExecutionClientCore,
9237 context: &AckedOrderProbeContext,
9238 sweep: &OrderReportSweep,
9239) {
9240 for (index, report) in &sweep.orders {
9241 if !core
9242 .cache()
9243 .client_order_id(&report.venue_order_id)
9244 .is_some_and(|cloid| context.dispatch.has_pending_modify(cloid))
9245 {
9246 restore_reconciled_order(
9247 core,
9248 &context.dispatch,
9249 report.venue_order_id,
9250 *index,
9251 report.order_status.is_closed(),
9252 );
9253 }
9254
9255 request_child_recovery(core, context, report);
9256 }
9257}
9258
9259fn request_child_recovery(
9260 core: &ExecutionClientCore,
9261 context: &AckedOrderProbeContext,
9262 report: &OrderStatusReport,
9263) {
9264 let cache = core.cache();
9265
9266 let Some(cloid) = cache.client_order_id(&report.venue_order_id).copied() else {
9267 return;
9268 };
9269
9270 let Some(order) = cache.order(&cloid) else {
9271 return;
9272 };
9273
9274 if order.parent_order_id().is_some()
9275 && !order.is_closed()
9276 && (report.order_status.is_closed() || report.filled_qty != order.filled_qty())
9277 {
9278 spawn_child_recovery(cloid, context);
9279 }
9280}
9281
9282fn spawn_child_recovery(cloid: ClientOrderId, context: &AckedOrderProbeContext) {
9283 let epoch = context.connection_epoch.load(Ordering::Acquire);
9284 {
9285 let mut events = context.dispatch.order_events.lock();
9286
9287 let Some(identity) = context.dispatch.order_identity(&cloid) else {
9288 return;
9289 };
9290
9291 if identity.child_fill_baseline().is_none()
9292 || identity.has_inferred_fills()
9293 || identity.has_child_publication()
9294 || context.dispatch.has_pending_modify(&cloid)
9295 {
9296 return;
9297 }
9298
9299 let deferred = events.entry(cloid).or_default();
9300 if deferred.recovery_epoch == Some(epoch) {
9301 return;
9302 }
9303
9304 deferred.recovery_epoch = Some(epoch);
9305 }
9306
9307 let context = context.clone();
9308 let tasks = context.pending_tasks.clone();
9309 let dispatch = context.dispatch.clone();
9310
9311 let future = async move {
9312 for attempt in 0..ACKED_ORDER_PROBE_ATTEMPTS {
9313 if context.connection_epoch.load(Ordering::Acquire) != epoch {
9314 break;
9315 }
9316
9317 let result = tokio::select! {
9318 () = context.cancellation_token.cancelled() => break,
9319 result = recover_child_order(cloid, epoch, &context) => result,
9320 };
9321
9322 match result {
9323 Ok(true) => break,
9324 Ok(false) => {}
9325 Err(e) => log::warn!(
9326 "Lighter child recovery unavailable for {cloid}: {}",
9327 scrub_auth(&format!("{e:#}"))
9328 ),
9329 }
9330
9331 if attempt + 1 < ACKED_ORDER_PROBE_ATTEMPTS {
9332 tokio::select! {
9333 () = context.cancellation_token.cancelled() => break,
9334 () = tokio::time::sleep(ACKED_ORDER_LOOKUP_DELAY) => {},
9335 }
9336 }
9337 }
9338
9339 let mut events = context.dispatch.order_events.lock();
9340 if let Some(deferred) = events.get_mut(&cloid)
9341 && deferred.recovery_epoch == Some(epoch)
9342 {
9343 deferred.recovery_epoch = None;
9344 }
9345 };
9346
9347 if let Err(e) = tasks.spawn(future) {
9348 if let Some(deferred) = dispatch.order_events.lock().get_mut(&cloid) {
9349 deferred.recovery_epoch = None;
9350 }
9351
9352 log::debug!("Skipping Lighter child recovery during shutdown: {e}");
9353 }
9354}
9355
9356async fn recover_child_order(
9357 cloid: ClientOrderId,
9358 epoch: u64,
9359 context: &AckedOrderProbeContext,
9360) -> anyhow::Result<bool> {
9361 let Some(identity) = context.dispatch.order_identity(&cloid) else {
9362 return Ok(true);
9363 };
9364
9365 if context.dispatch.has_pending_modify(&cloid)
9366 || identity.has_inferred_fills()
9367 || identity.has_child_publication()
9368 {
9369 return Ok(true);
9370 }
9371
9372 let Some(venue_order_id) = identity.venue_order_id() else {
9373 return Ok(false);
9374 };
9375
9376 let lookup = OrderReportLookup::new(
9377 &context.registry,
9378 &context.dispatch,
9379 Some(identity.instrument_id),
9380 Some(&cloid),
9381 Some(&venue_order_id),
9382 )?;
9383
9384 let Some(order) = collect_order(
9385 &context.http_client,
9386 &context.registry,
9387 &context.credential,
9388 &lookup,
9389 )
9390 .await?
9391 else {
9392 return Ok(false);
9393 };
9394
9395 anyhow::ensure!(
9396 order.owner_account_index == context.credential.account_index()
9397 && context.registry.instrument_id(order.market_index) == Some(identity.instrument_id)
9398 && identity.matches_venue_order_id(VenueOrderId::new(&order.order_id)),
9399 "Lighter child recovery ownership mismatch"
9400 );
9401 let instrument = LIGHTER_INSTRUMENT_CACHE
9402 .get(&identity.instrument_id)
9403 .context("missing child recovery instrument")?
9404 .value()
9405 .clone();
9406 let report = parse_ws_order_status_report(
9407 &order,
9408 &instrument,
9409 context.account_id,
9410 context.clock.get_time_ns(),
9411 )?;
9412
9413 if !report.order_status.is_closed() {
9414 return Ok(false);
9415 }
9416
9417 let mut instruments = AHashMap::new();
9418 instruments.insert(identity.instrument_id, instrument.clone());
9419
9420 let command = GenerateFillReports::new(
9421 UUID4::new(),
9422 context.clock.get_time_ns(),
9423 Some(identity.instrument_id),
9424 Some(venue_order_id),
9425 None,
9426 None,
9427 None,
9428 None,
9429 );
9430 let sweep = collect_fill_reports(
9431 &context.http_client,
9432 &context.registry,
9433 &context.credential,
9434 context.account_id,
9435 context.clock,
9436 &command,
9437 &instruments,
9438 &mut BTreeSet::new(),
9439 Some(order.order_index),
9440 )
9441 .await?;
9442 let mut events = context.dispatch.order_events.lock();
9443 if context.connection_epoch.load(Ordering::Acquire) != epoch
9444 || context.dispatch.has_pending_modify(&cloid)
9445 || identity.has_inferred_fills()
9446 || identity.has_child_publication()
9447 {
9448 return Ok(true);
9449 }
9450
9451 let deferred = events.entry(cloid).or_default();
9452 deferred.store_order(&order);
9453
9454 for mut fill in sweep.reports {
9455 fill.client_order_id = Some(cloid);
9456 if fill.venue_order_id != venue_order_id {
9457 continue;
9458 }
9459
9460 if let Some(fill) = child_fill_from_report(&fill, &identity, context.emitter.trader_id()) {
9461 context.dispatch.defer_fill(deferred, fill);
9462 }
9463 }
9464
9465 publish_child_recovery(cloid, &mut events, &context.dispatch, &context.emitter);
9466 Ok(identity.has_child_publication())
9467}
9468
9469fn child_fill_from_report(
9470 report: &FillReport,
9471 identity: &OrderIdentity,
9472 trader_id: TraderId,
9473) -> Option<OrderFilled> {
9474 let instrument = LIGHTER_INSTRUMENT_CACHE.get(&identity.instrument_id)?;
9475 let cloid = report.client_order_id.filter(|_| {
9476 identity.matches_venue_order_id(report.venue_order_id)
9477 && identity.instrument_id == report.instrument_id
9478 })?;
9479
9480 Some(OrderFilled::new(
9481 trader_id,
9482 identity.strategy_id,
9483 identity.instrument_id,
9484 cloid,
9485 report.venue_order_id,
9486 report.account_id,
9487 report.trade_id,
9488 report.order_side,
9489 identity.order_type,
9490 report.last_qty,
9491 report.last_px,
9492 instrument.quote_currency(),
9493 report.liquidity_side,
9494 UUID4::new(),
9495 report.ts_event,
9496 report.ts_init,
9497 false,
9498 None,
9499 Some(report.commission),
9500 None,
9501 ))
9502}
9503
9504fn publish_child_recovery(
9505 cloid: ClientOrderId,
9506 events: &mut AHashMap<ClientOrderId, DeferredOrderEvents>,
9507 dispatch: &WsDispatchState,
9508 emitter: &ExecutionEventEmitter,
9509) {
9510 let Some(identity) = dispatch.order_identity(&cloid) else {
9511 return;
9512 };
9513
9514 if position_modify_pending(&identity, dispatch) {
9515 return;
9516 }
9517
9518 let Some((ids, children)) = collect_child_recoveries(&identity, events, dispatch, emitter)
9519 else {
9520 return;
9521 };
9522
9523 let mut fills = Vec::new();
9524 let mut terminals = Vec::new();
9525
9526 for id in ids {
9527 let Some(mut deferred) = events.remove(&id) else {
9528 continue;
9529 };
9530
9531 fills.extend(deferred.fills.drain(..).map(|fill| {
9532 let sequence = deferred.fill_sequence.remove(&fill.trade_id).unwrap_or(0);
9533 (fill, sequence)
9534 }));
9535
9536 terminals.push((id, deferred));
9537 }
9538
9539 publish_child_updates(&children, &terminals, emitter);
9540
9541 fills.sort_by_key(|(fill, sequence)| (fill.ts_event, *sequence));
9542 for (fill, _) in fills {
9543 if dispatch.mark_trade_seen(fill.trade_id) {
9544 dispatch_lighter_fill(fill, emitter, dispatch, events);
9545 }
9546 }
9547
9548 for child in children {
9549 let id = child.cloid;
9550 let bundle = ExecutionReport::OrderWithFills(Box::new(child.report.clone()), Vec::new());
9551
9552 if let Err(e) = emitter.try_send_execution_report(bundle) {
9553 log::warn!("Lighter child report publication failed for {id}: {e}");
9554
9555 let Some(index) = terminals.iter().position(|(cid, _)| *cid == id) else {
9556 continue;
9557 };
9558
9559 let (_, deferred) = terminals.swap_remove(index);
9560 events.insert(id, deferred);
9561
9562 continue;
9563 }
9564
9565 child.identity.publish_child(&child.report, child.trade_ids);
9566 dispatch.retire_order_identity(&id);
9567 }
9568
9569 for (id, deferred) in terminals {
9570 if dispatch
9571 .order_identity(&id)
9572 .is_some_and(|identity| identity.child_fill_baseline().is_none())
9573 {
9574 flush_order_terminal(id, deferred, dispatch, emitter, events);
9575 }
9576 }
9577}
9578
9579struct ChildRecovery {
9580 cloid: ClientOrderId,
9581 identity: OrderIdentity,
9582 report: OrderStatusReport,
9583 trade_ids: Vec<TradeId>,
9584}
9585
9586fn collect_child_recoveries(
9587 identity: &OrderIdentity,
9588 events: &AHashMap<ClientOrderId, DeferredOrderEvents>,
9589 dispatch: &WsDispatchState,
9590 emitter: &ExecutionEventEmitter,
9591) -> Option<(Vec<ClientOrderId>, Vec<ChildRecovery>)> {
9592 let mut children = Vec::new();
9593 let mut ids = Vec::new();
9594
9595 for (id, deferred) in events {
9596 let Some(other) = dispatch.order_identity(id) else {
9597 continue;
9598 };
9599
9600 if other.strategy_id != identity.strategy_id
9601 || other.instrument_id != identity.instrument_id
9602 {
9603 continue;
9604 }
9605
9606 if dispatch.has_pending_modify(id) {
9607 return None;
9608 }
9609
9610 if other.child_fill_baseline().is_none() {
9611 ids.push(*id);
9612 continue;
9613 }
9614
9615 if deferred.fills.is_empty()
9616 && deferred
9617 .order
9618 .as_ref()
9619 .is_none_or(|order| !order.status.is_terminal())
9620 {
9621 continue;
9622 }
9623
9624 let report = child_recovery_report(*id, events, dispatch, emitter)?;
9625 let trade_ids = deferred.fills.iter().map(|fill| fill.trade_id).collect();
9626 children.push(ChildRecovery {
9627 cloid: *id,
9628 identity: other,
9629 report,
9630 trade_ids,
9631 });
9632
9633 ids.push(*id);
9634 }
9635
9636 Some((ids, children))
9637}
9638
9639fn publish_child_updates(
9640 children: &[ChildRecovery],
9641 terminals: &[(ClientOrderId, DeferredOrderEvents)],
9642 emitter: &ExecutionEventEmitter,
9643) {
9644 for child in children {
9645 if child.trade_ids.is_empty() {
9646 continue;
9647 }
9648
9649 let Some(order) = terminals
9650 .iter()
9651 .find(|(cid, _)| *cid == child.cloid)
9652 .and_then(|(_, deferred)| deferred.order.as_ref())
9653 else {
9654 continue;
9655 };
9656
9657 let Some(instrument) = LIGHTER_INSTRUMENT_CACHE.get(&child.identity.instrument_id) else {
9658 continue;
9659 };
9660
9661 if let Ok(updated) = parse_lighter_order_updated(
9662 order,
9663 &instrument,
9664 &child.identity,
9665 child.cloid,
9666 emitter.account_id(),
9667 emitter.trader_id(),
9668 child.report.ts_init,
9669 ) {
9670 emitter.send_order_event(OrderEventAny::Updated(updated));
9671 }
9672 }
9673}
9674
9675fn position_fills_pending(
9676 identity: &OrderIdentity,
9677 events: &AHashMap<ClientOrderId, DeferredOrderEvents>,
9678 dispatch: &WsDispatchState,
9679) -> bool {
9680 position_modify_pending(identity, dispatch)
9681 || events.iter().any(|(id, deferred)| {
9682 !deferred.fills.is_empty()
9683 && dispatch.order_identity(id).is_some_and(|other| {
9684 other.strategy_id == identity.strategy_id
9685 && other.instrument_id == identity.instrument_id
9686 })
9687 })
9688}
9689
9690fn position_modify_pending(identity: &OrderIdentity, dispatch: &WsDispatchState) -> bool {
9691 dispatch.order_identities.iter().any(|entry| {
9692 entry.value().strategy_id == identity.strategy_id
9693 && entry.value().instrument_id == identity.instrument_id
9694 && dispatch.has_pending_modify(entry.key())
9695 })
9696}
9697
9698fn child_recovery_report(
9699 cloid: ClientOrderId,
9700 events: &AHashMap<ClientOrderId, DeferredOrderEvents>,
9701 dispatch: &WsDispatchState,
9702 emitter: &ExecutionEventEmitter,
9703) -> Option<OrderStatusReport> {
9704 let identity = dispatch.order_identity(&cloid)?;
9705 if identity.has_child_publication() || identity.has_inferred_fills() {
9706 return None;
9707 }
9708
9709 let baseline = identity.child_fill_baseline()?;
9710 let deferred = events.get(&cloid)?;
9711 let order = deferred.order.as_ref()?;
9712 if !identity.matches_venue_order_id(VenueOrderId::new(&order.order_id)) {
9713 return None;
9714 }
9715
9716 let instrument = LIGHTER_INSTRUMENT_CACHE.get(&identity.instrument_id)?;
9717 let report = parse_ws_order_status_report(
9718 order,
9719 &instrument,
9720 emitter.account_id(),
9721 get_atomic_clock_realtime().get_time_ns(),
9722 )
9723 .ok()?;
9724 let report = dispatch.translate_order_cloid(dispatch.apply_order_contingencies(order, report));
9725 if !report.order_status.is_closed() || report.filled_qty > report.quantity {
9726 return None;
9727 }
9728
9729 let mut total = baseline.quantity.as_decimal();
9730 for fill in &deferred.fills {
9731 if baseline.trade_ids.contains(&fill.trade_id) {
9732 continue;
9733 }
9734
9735 if fill.venue_order_id != report.venue_order_id
9736 || fill.instrument_id != report.instrument_id
9737 || fill.account_id != report.account_id
9738 || Some(fill.order_side) != report.order_side
9739 || fill.commission.is_none()
9740 {
9741 return None;
9742 }
9743
9744 total += fill.last_qty.as_decimal();
9745 }
9746
9747 (total == report.filled_qty.as_decimal()).then_some(report)
9748}
9749
9750fn dispatch_lighter_fill(
9751 filled: OrderFilled,
9752 emitter: &ExecutionEventEmitter,
9753 dispatch: &WsDispatchState,
9754 events: &AHashMap<ClientOrderId, DeferredOrderEvents>,
9755) {
9756 let cloid = filled.client_order_id;
9757 if let Some(identity) = dispatch.order_identity(&cloid)
9758 && !identity.publish_fill(&filled)
9759 {
9760 return;
9761 }
9762
9763 ensure_fill_triggered(&filled, emitter, dispatch);
9764 emitter.send_order_event(OrderEventAny::Filled(filled));
9765 release_terminal(cloid, false, dispatch, emitter, events);
9766}
9767
9768fn defer_terminal(
9769 order: &LighterOrder,
9770 event: ParsedOrderEvent,
9771 identity: &OrderIdentity,
9772 cloid: ClientOrderId,
9773 dispatch: &WsDispatchState,
9774) -> Option<ParsedOrderEvent> {
9775 if !matches!(
9776 event,
9777 ParsedOrderEvent::Canceled(_) | ParsedOrderEvent::Expired(_)
9778 ) {
9779 return Some(event);
9780 }
9781
9782 let mut pending = dispatch.terminals_pending.lock();
9783 if pending.contains_key(&cloid) {
9784 return None;
9785 }
9786
9787 if order.filled_base_amount <= identity.filled_qty() {
9788 return Some(event);
9789 }
9790
9791 pending.insert(
9792 cloid,
9793 PendingTerminal {
9794 order: order.clone(),
9795 event,
9796 retry_at: tokio::time::Instant::now() + TERMINAL_FILL_DELAY,
9797 attempts: 0,
9798 },
9799 );
9800
9801 None
9802}
9803
9804fn release_terminal(
9805 cloid: ClientOrderId,
9806 force: bool,
9807 dispatch: &WsDispatchState,
9808 emitter: &ExecutionEventEmitter,
9809 events: &AHashMap<ClientOrderId, DeferredOrderEvents>,
9810) {
9811 let Some(identity) = dispatch.order_identity(&cloid) else {
9812 return;
9813 };
9814
9815 let terminal = {
9816 let mut pending = dispatch.terminals_pending.lock();
9817
9818 let Some(terminal) = pending.get(&cloid) else {
9819 return;
9820 };
9821
9822 if (identity.child_fill_baseline().is_some() && !identity.has_inferred_fills())
9823 || dispatch.has_pending_modify(&cloid)
9824 || position_fills_pending(&identity, events, dispatch)
9825 {
9826 return;
9827 }
9828
9829 if !force && identity.filled_qty() < terminal.order.filled_base_amount {
9830 return;
9831 }
9832
9833 pending.remove(&cloid).expect("pending terminal exists")
9834 };
9835
9836 let missing_qty = terminal.order.filled_base_amount - identity.filled_qty();
9837 if missing_qty > Decimal::ZERO {
9838 log::error!(
9839 "Lighter terminal fill recovery exhausted for {cloid}, venue_order_id={}, missing quantity={missing_qty}; releasing terminal for reconciliation",
9840 terminal.order.order_id,
9841 );
9842 }
9843
9844 dispatch_tracked_order_event(
9845 terminal.event,
9846 cloid,
9847 VenueOrderId::new(&terminal.order.order_id),
9848 &identity,
9849 emitter.account_id(),
9850 emitter.trader_id(),
9851 emitter,
9852 dispatch,
9853 get_atomic_clock_realtime().get_time_ns(),
9854 );
9855}
9856
9857async fn run_terminal_recovery(
9858 context: AckedOrderProbeContext,
9859 producers_stopped: CancellationToken,
9860) {
9861 let mut interval = tokio::time::interval(Duration::from_millis(100));
9862 interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
9863 let mut immediate = false;
9864
9865 loop {
9866 tokio::select! {
9867 biased;
9868 () = context.cancellation_token.cancelled() => break,
9869 () = context.dispatch.terminals_recover.notified() => immediate = true,
9870 _ = interval.tick() => {},
9871 }
9872
9873 tokio::select! {
9874 biased;
9875 () = context.cancellation_token.cancelled() => break,
9876 () = context.dispatch.terminals_recover.notified() => {
9877 immediate = true;
9878 continue;
9879 }
9880 () = recover_terminals(&context, immediate) => {},
9881 }
9882 immediate = false;
9883 }
9884
9885 producers_stopped.cancelled().await;
9886
9887 let drain = async {
9888 for _ in 0..ACKED_ORDER_PROBE_ATTEMPTS {
9889 recover_terminals(&context, true).await;
9890 if context.dispatch.terminals_pending.lock().is_empty() {
9891 break;
9892 }
9893 }
9894 };
9895
9896 let _ = tokio::time::timeout(TERMINAL_SHUTDOWN_TIMEOUT, drain).await;
9897 let events = context.dispatch.order_events.lock();
9898 let remaining = context
9899 .dispatch
9900 .terminals_pending
9901 .lock()
9902 .keys()
9903 .copied()
9904 .collect::<Vec<_>>();
9905
9906 for cloid in remaining {
9907 release_terminal(cloid, true, &context.dispatch, &context.emitter, &events);
9908 }
9909}
9910
9911async fn recover_terminals(context: &AckedOrderProbeContext, immediate: bool) {
9912 let now = tokio::time::Instant::now();
9913 let candidates = context
9914 .dispatch
9915 .terminals_pending
9916 .lock()
9917 .iter()
9918 .filter(|(_, terminal)| immediate || terminal.retry_at <= now)
9919 .map(|(cloid, terminal)| {
9920 (
9921 *cloid,
9922 terminal.order.clone(),
9923 terminal.attempts >= ACKED_ORDER_PROBE_ATTEMPTS,
9924 )
9925 })
9926 .collect::<Vec<_>>();
9927
9928 stream::iter(candidates)
9929 .for_each_concurrent(4, |(cloid, order, exhausted)| async move {
9930 {
9931 let events = context.dispatch.order_events.lock();
9932 release_terminal(
9933 cloid,
9934 exhausted,
9935 &context.dispatch,
9936 &context.emitter,
9937 &events,
9938 );
9939
9940 if exhausted {
9941 return;
9942 }
9943
9944 let mut pending = context.dispatch.terminals_pending.lock();
9945
9946 let Some(terminal) = pending.get_mut(&cloid) else {
9947 return;
9948 };
9949
9950 terminal.attempts += 1;
9951 }
9952
9953 let result = tokio::time::timeout(
9954 TERMINAL_RECOVERY_TIMEOUT,
9955 recover_terminal_trades(order.market_index, order.order_index, context),
9956 )
9957 .await;
9958
9959 match result {
9960 Ok(Ok(())) => {}
9961 Ok(Err(e)) => log::warn!(
9962 "Lighter terminal trade lookup failed for {cloid}: {}",
9963 scrub_auth(&format!("{e:#}"))
9964 ),
9965 Err(_) => log::warn!("Lighter terminal trade lookup timed out for {cloid}"),
9966 }
9967
9968 let events = context.dispatch.order_events.lock();
9969 release_terminal(cloid, false, &context.dispatch, &context.emitter, &events);
9970
9971 let exhausted = {
9972 let mut pending = context.dispatch.terminals_pending.lock();
9973
9974 let Some(terminal) = pending.get_mut(&cloid) else {
9975 return;
9976 };
9977
9978 terminal.retry_at = tokio::time::Instant::now() + TERMINAL_FILL_DELAY;
9979 terminal.attempts >= ACKED_ORDER_PROBE_ATTEMPTS
9980 };
9981
9982 if exhausted {
9983 release_terminal(cloid, true, &context.dispatch, &context.emitter, &events);
9984 }
9985 })
9986 .await;
9987}
9988
9989async fn recover_filled_reports(sweep: &OrderReportSweep, context: &AckedOrderProbeContext) {
9990 for (index, report) in &sweep.orders {
9991 if report.order_status != OrderStatus::Filled
9992 || !context.dispatch.restore_cached_order(
9993 report.venue_order_id,
9994 *index,
9995 report.instrument_id,
9996 )
9997 {
9998 continue;
9999 }
10000
10001 let Some(cloid) = context
10002 .dispatch
10003 .resolve_client_order_index_for_venue(*index, report.venue_order_id)
10004 else {
10005 continue;
10006 };
10007
10008 let Some(identity) = context.dispatch.order_identity(&cloid) else {
10009 continue;
10010 };
10011
10012 if identity.child_fill_baseline().is_some()
10013 || identity.has_inferred_fills()
10014 || report.filled_qty.as_decimal() <= identity.filled_qty()
10015 {
10016 continue;
10017 }
10018
10019 let Some(market_index) = context.registry.market_index(&report.instrument_id) else {
10020 continue;
10021 };
10022
10023 let result = async {
10024 let order_index = report
10025 .venue_order_id
10026 .as_str()
10027 .parse::<i64>()
10028 .context("invalid Lighter filled order index")?;
10029 tokio::time::timeout(
10030 TERMINAL_RECOVERY_TIMEOUT,
10031 recover_terminal_trades(market_index, order_index, context),
10032 )
10033 .await
10034 .context("Lighter filled-order trade lookup timed out")?
10035 }
10036 .await;
10037
10038 if let Err(e) = result {
10039 log::warn!(
10040 "Lighter filled-order recovery unavailable for {cloid}: {}",
10041 scrub_auth(&format!("{e:#}"))
10042 );
10043 }
10044 }
10045}
10046
10047async fn recover_report_fills(
10048 order: &LighterOrder,
10049 context: &AckedOrderProbeContext,
10050) -> anyhow::Result<()> {
10051 if !order.status.is_terminal() {
10052 return Ok(());
10053 }
10054
10055 let venue = VenueOrderId::new(&order.order_id);
10056
10057 let Some(instrument_id) = context.registry.instrument_id(order.market_index) else {
10058 return Ok(());
10059 };
10060
10061 if !context
10062 .dispatch
10063 .restore_cached_order(venue, order.client_order_index, instrument_id)
10064 {
10065 return Ok(());
10066 }
10067
10068 let Some(cloid) = context
10069 .dispatch
10070 .resolve_client_order_index_for_venue(order.client_order_index, venue)
10071 else {
10072 return Ok(());
10073 };
10074
10075 let missing = (order.status == LighterOrderStatus::Filled
10076 || context.dispatch.order_identities.contains_key(&cloid))
10077 && context
10078 .dispatch
10079 .order_identity(&cloid)
10080 .is_some_and(|identity| {
10081 order.filled_base_amount > identity.filled_qty()
10082 && (order.status != LighterOrderStatus::Filled
10083 || !identity.has_inferred_fills())
10084 });
10085
10086 if !missing {
10087 return Ok(());
10088 }
10089
10090 if order.status == LighterOrderStatus::Filled {
10091 tokio::time::timeout(
10092 TERMINAL_RECOVERY_TIMEOUT,
10093 recover_terminal_trades(order.market_index, order.order_index, context),
10094 )
10095 .await
10096 .context("Lighter filled-order trade lookup timed out")??;
10097 } else {
10098 dispatch_lighter_order(
10099 order,
10100 &context.dispatch,
10101 &context.emitter,
10102 &context.registry,
10103 context.account_id,
10104 context.emitter.trader_id(),
10105 context.clock.get_time_ns(),
10106 );
10107 }
10108
10109 Ok(())
10110}
10111
10112async fn recover_terminal_trades(
10113 market_index: i64,
10114 order_index: i64,
10115 context: &AckedOrderProbeContext,
10116) -> anyhow::Result<()> {
10117 let auth = build_auth_token_for(&context.credential)?;
10118
10119 let mut query = Zeroizing::new(LighterTradesQuery {
10120 authorization: None,
10121 auth: Some(auth),
10122 market_id: Some(market_index),
10123 account_index: Some(context.credential.account_index()),
10124 order_index: Some(order_index),
10125 sort_by: LighterTradeSortBy::Timestamp,
10126 sort_dir: Some(LighterSortDirection::Desc),
10127 cursor: None,
10128 from_timestamp: None,
10129 ask_filter: None,
10130 role: None,
10131 trade_type: None,
10132 limit: LIGHTER_REST_PAGE_SIZE,
10133 aggregate: None,
10134 });
10135
10136 let mut cursors = AHashSet::new();
10137 let mut trades = Vec::new();
10138
10139 for page in 0..MAX_RECONCILIATION_PAGES {
10140 let response = context.http_client.get_trades(&query).await?;
10141 for trade in response.trades {
10142 let account = context.credential.account_index();
10143 anyhow::ensure!(
10144 trade.market_id == market_index
10145 && ((trade.bid_account_id == account && trade.bid_id == order_index)
10146 || (trade.ask_account_id == account && trade.ask_id == order_index)),
10147 "Lighter terminal recovery returned a trade outside the requested order"
10148 );
10149 trades.push(trade);
10150 }
10151
10152 match response.next_cursor.filter(|cursor| !cursor.is_empty()) {
10153 Some(cursor) => {
10154 anyhow::ensure!(
10155 cursors.insert(cursor.clone()),
10156 "Lighter terminal recovery repeated cursor"
10157 );
10158 anyhow::ensure!(
10159 page + 1 < MAX_RECONCILIATION_PAGES,
10160 "Lighter terminal recovery exceeded page limit"
10161 );
10162 query.cursor = Some(cursor);
10163 }
10164 None => break,
10165 }
10166 }
10167
10168 trades.sort_by_key(|trade| (trade.timestamp, trade.trade_id));
10169 for trade in trades {
10170 dispatch_lighter_trade(
10171 &trade,
10172 &context.dispatch,
10173 &context.emitter,
10174 &context.registry,
10175 context.account_id,
10176 context.emitter.trader_id(),
10177 Some(context.credential.account_index()),
10178 context.clock.get_time_ns(),
10179 );
10180 }
10181
10182 Ok(())
10183}
10184
10185fn ensure_fill_triggered(
10186 filled: &OrderFilled,
10187 emitter: &ExecutionEventEmitter,
10188 dispatch: &WsDispatchState,
10189) {
10190 if TRIGGERABLE_ORDER_TYPES.contains(&filled.order_type)
10191 && dispatch.mark_triggered_emitted(filled.client_order_id)
10192 {
10193 let triggered = OrderTriggered::new(
10194 filled.trader_id,
10195 filled.strategy_id,
10196 filled.instrument_id,
10197 filled.client_order_id,
10198 UUID4::new(),
10199 filled.ts_event,
10200 filled.ts_init,
10201 false,
10202 Some(filled.venue_order_id),
10203 Some(filled.account_id),
10204 );
10205 emitter.send_order_event(OrderEventAny::Triggered(triggered));
10206 }
10207}
10208
10209#[expect(
10213 clippy::too_many_arguments,
10214 reason = "shared cleanup point across the typed-event variants"
10215)]
10216#[expect(
10217 clippy::needless_pass_by_value,
10218 reason = "event is destructured into typed OrderEventAny variants that consume the payload"
10219)]
10220fn dispatch_tracked_order_event(
10221 event: ParsedOrderEvent,
10222 cloid: ClientOrderId,
10223 venue_order_id: VenueOrderId,
10224 identity: &OrderIdentity,
10225 account_id: AccountId,
10226 trader_id: TraderId,
10227 emitter: &ExecutionEventEmitter,
10228 dispatch: &WsDispatchState,
10229 ts_init: UnixNanos,
10230) {
10231 let is_terminal;
10232
10233 match event {
10234 ParsedOrderEvent::Accepted(e) => {
10235 if !dispatch.claim_accepted_emission(&cloid) {
10236 log::debug!("Skipping duplicate OrderAccepted for {cloid}");
10237 return;
10238 }
10239
10240 is_terminal = false;
10241 emitter.send_order_event(OrderEventAny::Accepted(e));
10242 }
10243 ParsedOrderEvent::Triggered(e) => {
10244 if !dispatch.mark_triggered_emitted(cloid) {
10245 log::debug!("Skipping duplicate OrderTriggered for {cloid}");
10246 return;
10247 }
10248
10249 ensure_accepted_emitted(
10250 cloid,
10251 venue_order_id,
10252 identity,
10253 account_id,
10254 trader_id,
10255 emitter,
10256 dispatch,
10257 ts_init,
10258 );
10259 is_terminal = false;
10260 emitter.send_order_event(OrderEventAny::Triggered(e));
10261 }
10262 ParsedOrderEvent::Updated(e) => {
10263 dispatch.clear_pending_modify_if_update(&cloid, e.quantity, e.price, e.trigger_price);
10268 is_terminal = false;
10269 identity.publish_update(e.event_id);
10270 emitter.send_order_event(OrderEventAny::Updated(e));
10271 }
10272 ParsedOrderEvent::UpdatedThenTriggered { updated, triggered } => {
10273 if !dispatch.mark_triggered_emitted(cloid) {
10274 log::debug!("Skipping duplicate OrderTriggered for {cloid}");
10275 return;
10276 }
10277
10278 ensure_accepted_emitted(
10279 cloid,
10280 venue_order_id,
10281 identity,
10282 account_id,
10283 trader_id,
10284 emitter,
10285 dispatch,
10286 ts_init,
10287 );
10288 dispatch.clear_pending_modify_if_update(
10289 &cloid,
10290 updated.quantity,
10291 updated.price,
10292 updated.trigger_price,
10293 );
10294 is_terminal = false;
10295 identity.publish_update(updated.event_id);
10296 emitter.send_order_event(OrderEventAny::Updated(updated));
10297 emitter.send_order_event(OrderEventAny::Triggered(triggered));
10298 }
10299 ParsedOrderEvent::Canceled(e) => {
10300 ensure_accepted_emitted(
10301 cloid,
10302 venue_order_id,
10303 identity,
10304 account_id,
10305 trader_id,
10306 emitter,
10307 dispatch,
10308 ts_init,
10309 );
10310 is_terminal = true;
10311 emitter.send_order_event(OrderEventAny::Canceled(e));
10312 }
10313 ParsedOrderEvent::Expired(e) => {
10314 ensure_accepted_emitted(
10315 cloid,
10316 venue_order_id,
10317 identity,
10318 account_id,
10319 trader_id,
10320 emitter,
10321 dispatch,
10322 ts_init,
10323 );
10324 is_terminal = true;
10325 emitter.send_order_event(OrderEventAny::Expired(e));
10326 }
10327 ParsedOrderEvent::Rejected(e) => {
10328 is_terminal = true;
10329 emitter.send_order_event(OrderEventAny::Rejected(e));
10330 }
10331 }
10332
10333 if is_terminal {
10334 dispatch.clear_pending_order_action(&cloid);
10335 dispatch.retire_order_identity(&cloid);
10336 }
10337}
10338
10339#[expect(
10342 clippy::too_many_arguments,
10343 reason = "synthesized events need the full identity context to populate the event"
10344)]
10345fn ensure_accepted_emitted(
10346 cloid: ClientOrderId,
10347 venue_order_id: VenueOrderId,
10348 identity: &OrderIdentity,
10349 account_id: AccountId,
10350 trader_id: TraderId,
10351 emitter: &ExecutionEventEmitter,
10352 dispatch: &WsDispatchState,
10353 ts_init: UnixNanos,
10354) {
10355 if !dispatch.claim_accepted_emission(&cloid) {
10356 return;
10357 }
10358
10359 let accepted = OrderAccepted::new(
10360 trader_id,
10361 identity.strategy_id,
10362 identity.instrument_id,
10363 cloid,
10364 venue_order_id,
10365 account_id,
10366 UUID4::new(),
10367 ts_init,
10368 ts_init,
10369 false,
10370 );
10371 emitter.send_order_event(OrderEventAny::Accepted(accepted));
10372}
10373
10374#[cfg(test)]
10375mod tests {
10376 use std::{
10377 cell::RefCell,
10378 rc::Rc,
10379 sync::{Arc, atomic::AtomicUsize},
10380 };
10381
10382 use axum::{
10383 Router,
10384 extract::Query,
10385 routing::{get, post},
10386 };
10387 use nautilus_common::{
10388 cache::Cache,
10389 clock::{Clock, VirtualClock},
10390 factories::OrderFactory,
10391 live::runner::replace_exec_event_sender,
10392 messages::{ExecutionEvent, ExecutionReport as EngineExecutionReport},
10393 testing::wait_until_async,
10394 };
10395 use nautilus_execution::engine::ExecutionEngine;
10396 use nautilus_live::manager::{ExecutionManager, ExecutionManagerConfig};
10397 use nautilus_model::{
10398 accounts::MarginAccount,
10399 data::QuoteTick,
10400 enums::{ContingencyType, LiquiditySide, OrderSide, OrderStatus, TimeInForce},
10401 events::{
10402 AccountState, OrderCanceled, OrderEventAny, OrderPendingCancel, OrderSubmitted,
10403 OrderTriggered,
10404 },
10405 identifiers::{
10406 InstrumentId, OrderListId, PositionId, StrategyId, Symbol, TradeId, TraderId,
10407 VenueOrderId,
10408 },
10409 instruments::CryptoPerpetual,
10410 orders::{LimitOrder, OrderList, OrderTestBuilder, stubs::TestOrderEventStubs},
10411 position::Position,
10412 types::{Currency, Money, Price},
10413 };
10414 use rstest::rstest;
10415
10416 use super::*;
10417 use crate::{
10418 common::{
10419 consts::{
10420 LIGHTER_NAUTILUS_INTEGRATOR_ACCOUNT_INDEX,
10421 LIGHTER_ROBINHOOD_NAUTILUS_INTEGRATOR_ACCOUNT_INDEX, LIGHTER_VENUE,
10422 },
10423 enums::{LighterDeployment, LighterEnvironment, LighterProductType},
10424 },
10425 http::models::{LighterNextNonce, LighterOrders, LighterTx},
10426 signing::tx::TX_HASH_BYTES,
10427 };
10428
10429 const TEST_PRIVATE_KEY: &str =
10430 "0b8e0f63c24d8baacd9d29ad4e9a4b73c4a8d2bb8b16dc4fa9d7c2e1d3a8b1f0e8d3a4c5b6e7f001";
10431 const TEST_ACCOUNT_INDEX: u64 = 12345;
10432 const TEST_ACCOUNT_INDEX_I64: i64 = 12345;
10433 const TEST_API_KEY_INDEX: u8 = 5;
10434 const TEST_NEXT_NONCE: i64 = 42;
10435 const TEST_MARKET_INDEX: i64 = 0;
10436 const TEST_ORDER_NONCE: i64 = 281_474_720_725_346;
10437 const TEST_SUBMISSION_NONCE: i64 = 2_042;
10438
10439 fn handle_send_tx_ack(
10440 dispatch: &WsDispatchState,
10441 account_index: Option<i64>,
10442 code: i64,
10443 tx_hash: Option<&str>,
10444 ) -> Option<PendingSendTx> {
10445 handle_send_tx_ack_for_connection(dispatch, account_index, 0, code, tx_hash)
10446 }
10447
10448 #[expect(
10449 clippy::too_many_arguments,
10450 reason = "test wrapper preserves the established rejection call shape"
10451 )]
10452 fn handle_send_tx_rejection(
10453 dispatch: &WsDispatchState,
10454 emitter: &ExecutionEventEmitter,
10455 account_index: Option<i64>,
10456 now: UnixNanos,
10457 source: SendTxRejectionSource,
10458 code: Option<i64>,
10459 message: &str,
10460 tx_hash: Option<&str>,
10461 ) -> bool {
10462 handle_send_tx_rejection_for_connection(
10463 dispatch,
10464 emitter,
10465 account_index,
10466 0,
10467 now,
10468 source,
10469 code,
10470 message,
10471 tx_hash,
10472 )
10473 }
10474
10475 fn trader_id() -> TraderId {
10476 TraderId::from("TRADER-001")
10477 }
10478
10479 fn client_id() -> ClientId {
10480 ClientId::from("LIGHTER")
10481 }
10482
10483 fn account_id() -> AccountId {
10484 AccountId::from("LIGHTER-001")
10485 }
10486
10487 fn strategy_id() -> StrategyId {
10488 StrategyId::from("S-001")
10489 }
10490
10491 fn test_credential() -> Credential {
10492 Credential::new(TEST_API_KEY_INDEX, TEST_PRIVATE_KEY, TEST_ACCOUNT_INDEX).unwrap()
10493 }
10494
10495 fn test_config() -> LighterExecutionClientConfig {
10496 test_config_with_use_gtd(true)
10497 }
10498
10499 fn test_config_with_use_gtd(use_gtd: bool) -> LighterExecutionClientConfig {
10500 LighterExecutionClientConfig {
10501 account_id: account_id(),
10502 account_index: Some(TEST_ACCOUNT_INDEX),
10503 api_key_index: Some(TEST_API_KEY_INDEX),
10504 private_key: Some(TEST_PRIVATE_KEY.into()),
10505 base_url_http: Some("http://127.0.0.1:1".to_string()),
10506 base_url_ws: Some("ws://127.0.0.1:1/stream".to_string()),
10507 proxy_url: None,
10508 environment: LighterEnvironment::Testnet,
10509 deployment: LighterDeployment::Lighter,
10510 venue: None,
10511 http_timeout_secs: 1,
10512 ws_timeout_secs: 1,
10513 market_order_slippage_bps: 50,
10514 rest_quota_per_min: None,
10515 sendtx_quota_per_min: None,
10516 transport_backend: Default::default(),
10517 use_gtd,
10518 scheduled_cancel_timeout_secs: None,
10519 }
10520 }
10521
10522 #[rstest]
10523 fn format_between_timestamps_uses_lighter_seconds_range() {
10524 let start = UnixNanos::from(1_700_000_000_123_456_789);
10525 let end = UnixNanos::from(1_700_003_600_987_654_321);
10526 let now = UnixNanos::from(1_700_007_200_000_000_000);
10527
10528 assert_eq!(
10529 format_between_timestamps(Some(start), Some(end), now),
10530 Some("1700000000-1700003600".to_string()),
10531 );
10532 assert_eq!(
10533 format_between_timestamps(Some(start), None, now),
10534 Some("1700000000-1700007200".to_string()),
10535 );
10536 assert_eq!(
10537 format_between_timestamps(None, Some(end), now),
10538 Some("0-1700003600".to_string()),
10539 );
10540 assert_eq!(format_between_timestamps(None, None, now), None);
10541 }
10542
10543 #[rstest]
10544 #[case::write_timeout(LighterWsError::Transport(SendError::WriteTimeout), "Ambiguous")]
10545 #[case::broken_pipe(
10546 LighterWsError::Transport(SendError::BrokenPipe("writer closed".to_string())),
10547 "Ambiguous",
10548 )]
10549 #[case::handler_result_lost(
10550 LighterWsError::SendTxOutcomeUnknown("result sender dropped".to_string()),
10551 "Ambiguous",
10552 )]
10553 #[case::network(LighterWsError::Network("disconnected".to_string()), "Ambiguous")]
10554 #[case::parse(LighterWsError::Parse("invalid ack".to_string()), "Ambiguous")]
10555 #[case::invalid_input(
10556 LighterWsError::Transport(SendError::InvalidInput("invalid payload".to_string())),
10557 "NotSent",
10558 )]
10559 #[case::buffer_full(LighterWsError::Transport(SendError::BufferFull), "NotSent")]
10560 #[case::closed(LighterWsError::Transport(SendError::Closed), "NotSent")]
10561 #[case::connection_changed(LighterWsError::Transport(SendError::ConnectionChanged), "NotSent")]
10562 #[case::wait_timeout(LighterWsError::Transport(SendError::Timeout), "NotSent")]
10563 #[case::authentication(
10564 LighterWsError::Authentication("invalid token".to_string()),
10565 "NotSent",
10566 )]
10567 #[case::client_unavailable(
10568 LighterWsError::Client("handler unavailable".to_string()),
10569 "NotSent",
10570 )]
10571 fn ws_command_failure_classifies_delivery_evidence(
10572 #[case] error: LighterWsError,
10573 #[case] expected: &str,
10574 ) {
10575 let failure = classify_lighter_ws_command_failure("submit_order", &error);
10576
10577 let actual = match &failure {
10578 CommandFailure::NotSent(_) => "NotSent",
10579 CommandFailure::Ambiguous(_) => "Ambiguous",
10580 CommandFailure::VenueRejected(_) => "VenueRejected",
10581 };
10582
10583 assert_eq!(actual, expected);
10584 assert!(command_failure_reason(&failure).contains("submit_order"));
10585 }
10586
10587 #[rstest]
10588 fn venue_rejection_reason_is_clean_and_bounded() {
10589 let oversized = "x".repeat(STRATEGY_REASON_MAX_CHARS + 100);
10590 let message = format!("<html>\n rejected\0 because {oversized}</html>");
10591
10592 let failure = venue_rejection_failure(Some(20_001), &message);
10593 let reason = command_failure_reason(&failure);
10594
10595 assert!(matches!(&failure, CommandFailure::VenueRejected(_)));
10596 assert!(!reason.contains('<'));
10597 assert!(!reason.contains('\0'));
10598 assert!(!reason.contains(" "));
10599 assert_eq!(reason.chars().count(), STRATEGY_REASON_MAX_CHARS);
10600 assert!(reason.starts_with("LIGHTER_20001: rejected because "));
10601 }
10602
10603 #[rstest]
10604 fn venue_rejection_reason_uses_stable_fallback_for_empty_markup() {
10605 let failure = venue_rejection_failure(Some(20_001), "<html></html>\n");
10606
10607 assert_eq!(
10608 failure,
10609 CommandFailure::VenueRejected("LIGHTER_20001".to_string()),
10610 );
10611 }
10612
10613 #[rstest]
10614 fn venue_rejection_reason_preserves_comparison_and_removes_format_controls() {
10615 let failure = venue_rejection_failure(
10616 Some(20_001),
10617 "price must be < 100\u{202e}\u{2066}\u{200b}<strong>now</strong>",
10618 );
10619
10620 assert_eq!(
10621 failure,
10622 CommandFailure::VenueRejected("LIGHTER_20001: price must be < 100 now".to_string(),),
10623 );
10624 }
10625
10626 #[rstest]
10627 fn unsupported_order_type_reason_keeps_lighter_capabilities() {
10628 assert_eq!(
10629 unsupported_lighter_order_type_reason(OrderType::MarketToLimit),
10630 "UNSUPPORTED_ORDER_TYPE: MARKET_TO_LIMIT; Lighter supports MARKET, LIMIT, STOP_MARKET, STOP_LIMIT, MARKET_IF_TOUCHED, and LIMIT_IF_TOUCHED",
10631 );
10632 }
10633
10634 #[rstest]
10635 fn mass_status_propagates_commission_error_and_downgrades_other_fill_errors() {
10636 let e = anyhow::Error::new(LighterCommissionError::new("invalid precision"))
10637 .context("failed to parse Lighter fill report");
10638
10639 let commission_error = fill_reports_for_mass_status(Err(e))
10640 .expect_err("commission construction must fail mass status");
10641 let ordinary_error = fill_reports_for_mass_status(Err(anyhow::anyhow!("transport failed")))
10642 .expect("ordinary source errors make mass status incomplete");
10643
10644 let complete = fill_reports_for_mass_status(Ok(FillSweep {
10645 reports: Vec::new(),
10646 covers_window: true,
10647 withheld_instruments: AHashSet::new(),
10648 }))
10649 .expect("complete fill sweep");
10650
10651 assert!(is_commission_error(&commission_error));
10652 assert!(ordinary_error.0.is_empty());
10653 assert!(!ordinary_error.1);
10654 assert!(complete.0.is_empty());
10655 assert!(complete.1);
10656 }
10657
10658 fn create_execution_client() -> (
10659 LighterExecutionClient,
10660 Rc<RefCell<Cache>>,
10661 tokio::sync::mpsc::UnboundedReceiver<ExecutionEvent>,
10662 ) {
10663 create_execution_client_with_config(test_config())
10664 }
10665
10666 #[rstest]
10667 #[case(419)]
10668 #[case(1_295_941)]
10669 #[case(u64::MAX)]
10670 fn scheduled_cancel_constructor_rejects_invalid_timeout(#[case] timeout: u64) {
10671 let (client, _, _) = create_execution_client();
10672 let mut config = test_config();
10673 config.scheduled_cancel_timeout_secs = Some(timeout);
10674 let error = LighterExecutionClient::new(client.core, config).unwrap_err();
10675 assert_eq!(
10676 error.to_string(),
10677 "scheduled_cancel_timeout_secs must be in [420, 1295940]"
10678 );
10679 }
10680
10681 #[tokio::test]
10682 async fn scheduled_cancel_timeout_preserves_non_tail_nonce() {
10683 let (mut client, _, _) = create_execution_client();
10684 client.tx_rate_limiter = build_tx_rate_limiter(Some(1));
10685 let key = ustr::Ustr::from(crate::common::rate_limit::LIGHTER_TX_BUCKET);
10686 assert!(client.tx_rate_limiter.check_key(&key).is_ok());
10687 let context = client
10688 .fanout_dispatch_context(client.credential.as_ref().unwrap())
10689 .unwrap();
10690
10691 let later = {
10692 let renewal = context.refresh_scheduled_cancel(420);
10693 tokio::pin!(renewal);
10694 tokio::select! {
10695 result = &mut renewal => panic!("renewal must wait for quota: {result:?}"),
10696 () = tokio::time::sleep(Duration::from_millis(20)) => {},
10697 }
10698 let later = context.build_tx_context().unwrap();
10699 assert_eq!(later.context.nonce, TEST_NEXT_NONCE + 1);
10700 assert!(
10701 tokio::time::timeout(Duration::from_millis(50), &mut renewal)
10702 .await
10703 .is_err()
10704 );
10705 later
10706 };
10707
10708 assert!(
10709 tokio::time::timeout(
10710 Duration::from_millis(150),
10711 later.send_reservation.wait_for_turn()
10712 )
10713 .await
10714 .is_err()
10715 );
10716 assert!(!client.pending_tasks.all_finished());
10717 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
10718 assert_eq!(
10719 client
10720 .dispatch
10721 .nonce_manager
10722 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
10723 Some(TEST_NEXT_NONCE + 1)
10724 );
10725 drop(later);
10726 client.begin_session_shutdown();
10727 client.finish_session_shutdown().await.unwrap();
10728 assert!(client.pending_tasks.is_empty());
10729 }
10730
10731 #[tokio::test]
10732 async fn scheduled_cancel_timeout_cancels_quota_wait_and_releases_nonce() {
10733 let (mut client, _, _) = create_execution_client();
10734 client.tx_rate_limiter = build_tx_rate_limiter(Some(1));
10735 let key = ustr::Ustr::from(crate::common::rate_limit::LIGHTER_TX_BUCKET);
10736 assert!(client.tx_rate_limiter.check_key(&key).is_ok());
10737 let context = client
10738 .fanout_dispatch_context(client.credential.as_ref().unwrap())
10739 .unwrap();
10740 assert!(
10741 tokio::time::timeout(
10742 Duration::from_millis(50),
10743 context.refresh_scheduled_cancel(420)
10744 )
10745 .await
10746 .is_err()
10747 );
10748 tokio::time::timeout(Duration::from_secs(2), async {
10749 while !client.pending_tasks.all_finished() {
10750 tokio::task::yield_now().await;
10751 }
10752 })
10753 .await
10754 .unwrap();
10755
10756 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
10757 assert_eq!(
10758 client
10759 .dispatch
10760 .nonce_manager
10761 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
10762 .unwrap(),
10763 TEST_NEXT_NONCE
10764 );
10765 client.begin_session_shutdown();
10766 client.finish_session_shutdown().await.unwrap();
10767 assert!(client.pending_tasks.is_empty());
10768 }
10769
10770 #[rstest]
10771 #[tokio::test]
10772 async fn account_cancellation_dropped_during_quota_wait_is_retained_until_shutdown() {
10773 let (mut client, _, _) = create_execution_client();
10774 client.tx_rate_limiter = build_tx_rate_limiter(Some(1));
10775 let key = ustr::Ustr::from(crate::common::rate_limit::LIGHTER_TX_BUCKET);
10776 assert!(client.tx_rate_limiter.check_key(&key).is_ok());
10777 {
10778 let pending = client.cancel_all_orders_account();
10779 tokio::pin!(pending);
10780 tokio::select! {
10781 result = &mut pending => panic!("control must wait for quota: {result:?}"),
10782 () = tokio::time::sleep(Duration::from_millis(50)) => {}
10783 }
10784 }
10785
10786 assert!(!client.pending_tasks.is_empty());
10787 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
10788
10789 client.begin_session_shutdown();
10790 client.finish_session_shutdown().await.unwrap();
10791
10792 assert!(client.pending_tasks.is_empty());
10793 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
10794 assert_eq!(
10795 client
10796 .dispatch
10797 .nonce_manager
10798 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
10799 .unwrap(),
10800 TEST_NEXT_NONCE,
10801 );
10802 }
10803
10804 #[rstest]
10805 #[tokio::test]
10806 async fn account_cancellation_spawn_rejection_releases_latest_nonce() {
10807 let (client, _, _) = create_execution_client();
10808 client.pending_tasks.begin_shutdown();
10809
10810 let error = client.cancel_all_orders_account().await.unwrap_err();
10811
10812 assert_eq!(
10813 error.to_string(),
10814 "failed to spawn Lighter account cancellation"
10815 );
10816 assert!(client.pending_tasks.is_empty());
10817 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
10818 assert_eq!(
10819 client
10820 .dispatch
10821 .nonce_manager
10822 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
10823 .unwrap(),
10824 TEST_NEXT_NONCE,
10825 );
10826 }
10827
10828 #[rstest]
10829 #[tokio::test]
10830 async fn account_cancellation_ambiguous_error_identifies_pending_transaction() {
10831 let (mut client, _, _) = create_execution_client();
10832 client.ws_client.drop_next_send_tx_result_for_test().await;
10833
10834 let error = client.cancel_all_orders_account().await.unwrap_err();
10835 let pending = client.dispatch.take_pending_sendtx();
10836
10837 assert_eq!(pending.len(), 1);
10838 assert_eq!(pending[0].nonce, TEST_NEXT_NONCE);
10839 assert_eq!(pending[0].api_key_index, TEST_API_KEY_INDEX);
10840 assert_eq!(
10841 error.to_string(),
10842 format!(
10843 "failed to submit Lighter account cancellation transaction {}",
10844 pending[0].tx_hash
10845 ),
10846 );
10847 assert!(matches!(
10848 error.downcast_ref::<LighterWsError>(),
10849 Some(LighterWsError::SendTxOutcomeUnknown(_)),
10850 ));
10851 assert_eq!(
10852 client
10853 .dispatch
10854 .nonce_manager
10855 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
10856 .unwrap(),
10857 TEST_NEXT_NONCE + 1,
10858 );
10859 client.begin_session_shutdown();
10860 client.finish_session_shutdown().await.unwrap();
10861 }
10862
10863 #[rstest]
10864 #[case::definite_failure(false, 0, TEST_NEXT_NONCE + 1)]
10865 #[case::ambiguous_failure(true, 1, TEST_NEXT_NONCE + 2)]
10866 #[tokio::test]
10867 async fn account_cancellation_dropped_caller_preserves_dispatch_failure_handling(
10868 #[case] ambiguous: bool,
10869 #[case] pending_count: usize,
10870 #[case] next_nonce: i64,
10871 ) {
10872 let (mut client, _, _) = create_execution_client();
10873 if ambiguous {
10874 client.ws_client.drop_next_send_tx_result_for_test().await;
10875 }
10876
10877 let mut blocker = client.build_tx_context(&test_credential()).unwrap();
10878 {
10879 let pending = client.cancel_all_orders_account();
10880 tokio::pin!(pending);
10881 tokio::select! {
10882 result = &mut pending => panic!("control must wait for its turn: {result:?}"),
10883 () = tokio::time::sleep(Duration::from_millis(50)) => {}
10884 }
10885 }
10886
10887 assert!(!client.pending_tasks.is_empty());
10888 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
10889 blocker.send_reservation.release();
10890 wait_until_async(
10891 || async { client.pending_tasks.is_empty() },
10892 Duration::from_secs(2),
10893 )
10894 .await;
10895
10896 assert_eq!(client.dispatch.pending_sendtx_len(), pending_count);
10897 assert_eq!(
10898 client
10899 .dispatch
10900 .nonce_manager
10901 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
10902 .unwrap(),
10903 next_nonce,
10904 );
10905 client.begin_session_shutdown();
10906 client.finish_session_shutdown().await.unwrap();
10907 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
10908 }
10909
10910 #[rstest]
10911 #[case::first_start(false)]
10912 #[case::already_started(true)]
10913 fn execution_sender_is_available_before_start_and_replaced_on_start(#[case] started: bool) {
10914 let (client, _cache, _rx) = create_execution_client();
10915 let (first_sender, mut first_receiver) = tokio::sync::mpsc::unbounded_channel();
10916 replace_exec_event_sender(first_sender);
10917 let mut client = LighterExecutionClient::new(client.core, test_config()).unwrap();
10918 let emitter = client.emitter.clone();
10919
10920 let submitted = OrderSubmitted::new(
10921 trader_id(),
10922 strategy_id(),
10923 InstrumentId::from("ETH-PERP.LIGHTER"),
10924 ClientOrderId::from("BEFORE-START"),
10925 account_id(),
10926 UUID4::new(),
10927 UnixNanos::from(17),
10928 UnixNanos::from(23),
10929 );
10930 emitter
10931 .try_send_order_event(OrderEventAny::Submitted(submitted.clone()))
10932 .unwrap();
10933
10934 if started {
10935 client.start().unwrap();
10936 }
10937
10938 let (second_sender, mut second_receiver) = tokio::sync::mpsc::unbounded_channel();
10939 replace_exec_event_sender(second_sender);
10940 client.start().unwrap();
10941
10942 let after_start = OrderSubmitted::new(
10943 trader_id(),
10944 strategy_id(),
10945 InstrumentId::from("BTC-PERP.LIGHTER"),
10946 ClientOrderId::from("AFTER-START"),
10947 account_id(),
10948 UUID4::new(),
10949 UnixNanos::from(29),
10950 UnixNanos::from(31),
10951 );
10952 emitter
10953 .try_send_order_event(OrderEventAny::Submitted(after_start.clone()))
10954 .unwrap();
10955
10956 for (receiver, expected) in [
10957 (&mut first_receiver, submitted),
10958 (&mut second_receiver, after_start),
10959 ] {
10960 let ExecutionEvent::Order(OrderEventAny::Submitted(actual)) =
10961 receiver.try_recv().unwrap()
10962 else {
10963 panic!("expected submitted event");
10964 };
10965
10966 assert_eq!(actual, expected);
10967 assert!(receiver.try_recv().is_err());
10968 }
10969 }
10970
10971 fn create_execution_client_with_config(
10972 config: LighterExecutionClientConfig,
10973 ) -> (
10974 LighterExecutionClient,
10975 Rc<RefCell<Cache>>,
10976 tokio::sync::mpsc::UnboundedReceiver<ExecutionEvent>,
10977 ) {
10978 let cache = Rc::new(RefCell::new(Cache::default()));
10979 let venue = config.resolved_venue();
10980
10981 let core = ExecutionClientCore::new(
10982 trader_id(),
10983 client_id(),
10984 venue,
10985 OmsType::Netting,
10986 account_id(),
10987 AccountType::Margin,
10988 None,
10989 cache.clone(),
10990 );
10991
10992 let mut client = LighterExecutionClient::new(core, config).unwrap();
10993 client.dispatch.nonce_manager.refresh(
10994 TEST_ACCOUNT_INDEX_I64,
10995 TEST_API_KEY_INDEX,
10996 TEST_NEXT_NONCE,
10997 );
10998
10999 let (sender, receiver) = tokio::sync::mpsc::unbounded_channel::<ExecutionEvent>();
11000 client.emitter.set_sender(sender);
11001
11002 (client, cache, receiver)
11003 }
11004
11005 #[rstest]
11006 #[case::inline_deferred(false, true)]
11007 #[case::worker_deferred(true, true)]
11008 #[case::inline_absent(false, false)]
11009 #[case::worker_absent(true, false)]
11010 #[tokio::test]
11011 async fn single_order_lookup_distinguishes_deferred_report_from_absence(
11012 #[case] worker: bool,
11013 #[case] present: bool,
11014 ) {
11015 let rig = dispatcher_rig("DEFERRED-PROBE");
11016 register_identity(&rig);
11017 let order = dispatcher_test_order(&rig, LighterOrderStatus::Open);
11018 let venue_order_id = VenueOrderId::from(order.order_id.as_str());
11019 rig.dispatch.observe_create_submission(
11020 &rig.cloid,
11021 order.client_order_index,
11022 TEST_SUBMISSION_NONCE,
11023 venue_order_id,
11024 );
11025 rig.dispatch
11026 .order_identity(&rig.cloid)
11027 .unwrap()
11028 .publish_update(UUID4::new());
11029
11030 let response = LighterOrders {
11031 code: 200,
11032 message: None,
11033 next_cursor: None,
11034 orders: if present { vec![order] } else { Vec::new() },
11035 };
11036
11037 let body = serde_json::to_string(&response).unwrap();
11038 let app = Router::new()
11039 .route(
11040 "/api/v1/accountActiveOrders",
11041 get(move || {
11042 let body = body.clone();
11043 async move { body }
11044 }),
11045 )
11046 .route(
11047 "/api/v1/accountInactiveOrders",
11048 get(|| async {
11049 axum::Json(LighterOrders {
11050 code: 200,
11051 message: None,
11052 next_cursor: None,
11053 orders: Vec::new(),
11054 })
11055 }),
11056 );
11057
11058 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
11059 let addr = listener.local_addr().unwrap();
11060 let server = tokio::spawn(async move {
11061 axum::serve(listener, app).await.unwrap();
11062 });
11063
11064 let config = LighterExecutionClientConfig {
11065 base_url_http: Some(format!("http://{addr}")),
11066 ..test_config()
11067 };
11068
11069 let (mut client, _cache, mut rx) = create_execution_client_with_config(config);
11070 client.registry = rig.registry;
11071 client.dispatch = rig.dispatch;
11072
11073 let command = GenerateOrderStatusReport::new(
11074 UUID4::new(),
11075 UnixNanos::from(1),
11076 Some(rig.instrument_id),
11077 Some(rig.cloid),
11078 Some(venue_order_id),
11079 None,
11080 None,
11081 );
11082
11083 let result = if worker {
11084 let task = client.generate_order_status_report_task(&command).unwrap();
11085 task.collection.await;
11086 (task.finish)()
11087 } else {
11088 client.generate_order_status_report(&command).await
11089 };
11090
11091 server.abort();
11092 let _ = server.await;
11093
11094 if present {
11095 assert_eq!(
11096 result.unwrap_err().to_string(),
11097 "Lighter order report deferred until execution state is applied"
11098 );
11099 } else {
11100 assert_eq!(result.unwrap(), None);
11101 }
11102
11103 assert!(
11104 client
11105 .dispatch
11106 .order_identity(&rig.cloid)
11107 .unwrap()
11108 .has_publication()
11109 );
11110 assert!(rx.try_recv().is_err());
11111 }
11112
11113 fn register_test_instrument(
11114 client: &LighterExecutionClient,
11115 cache: &Rc<RefCell<Cache>>,
11116 ) -> InstrumentId {
11117 let instrument_id =
11118 client
11119 .registry
11120 .insert(TEST_MARKET_INDEX, "ETH", LighterProductType::Perp);
11121 let instrument = InstrumentAny::CryptoPerpetual(
11122 CryptoPerpetual::builder()
11123 .instrument_id(instrument_id)
11124 .raw_symbol(Symbol::new("ETH-PERP"))
11125 .base_currency(Currency::from("ETH"))
11126 .quote_currency(Currency::from("USDC"))
11127 .settlement_currency(Currency::from("USDC"))
11128 .is_inverse(false)
11129 .price_precision(2)
11130 .size_precision(4)
11131 .price_increment(Price::from("0.01"))
11132 .size_increment(Quantity::from("0.0001"))
11133 .min_notional(Money::from("10.000000 USDC"))
11134 .ts_event(UnixNanos::default())
11135 .ts_init(UnixNanos::default())
11136 .build()
11137 .unwrap(),
11138 );
11139
11140 cache.borrow_mut().add_instrument(instrument).unwrap();
11141
11142 instrument_id
11143 }
11144
11145 fn test_order_factory() -> OrderFactory {
11146 let clock = Rc::new(RefCell::new(VirtualClock::new()));
11147
11148 OrderFactory::new(
11149 trader_id(),
11150 strategy_id(),
11151 Some(0),
11152 Some(0),
11153 clock,
11154 false,
11155 false,
11156 )
11157 }
11158
11159 fn test_limit_order(
11160 factory: &mut OrderFactory,
11161 instrument_id: InstrumentId,
11162 client_order_id: &str,
11163 ) -> OrderAny {
11164 test_limit_order_with(
11165 factory,
11166 instrument_id,
11167 client_order_id,
11168 OrderSide::Buy,
11169 TimeInForce::Gtc,
11170 None,
11171 false,
11172 )
11173 }
11174
11175 fn test_limit_order_with(
11176 factory: &mut OrderFactory,
11177 instrument_id: InstrumentId,
11178 client_order_id: &str,
11179 side: OrderSide,
11180 tif: TimeInForce,
11181 expire_time: Option<UnixNanos>,
11182 reduce_only: bool,
11183 ) -> OrderAny {
11184 factory.limit(
11185 instrument_id,
11186 side,
11187 Quantity::from("0.1000"),
11188 Price::from("2361.31"),
11189 Some(tif),
11190 expire_time,
11191 Some(false),
11192 Some(reduce_only),
11193 None,
11194 None,
11195 None,
11196 None,
11197 None,
11198 None,
11199 None,
11200 Some(ClientOrderId::from(client_order_id)),
11201 )
11202 }
11203
11204 fn cache_order(cache: &Rc<RefCell<Cache>>, order: OrderAny) {
11205 cache
11206 .borrow_mut()
11207 .add_order(order, None, Some(client_id()), false)
11208 .unwrap();
11209 }
11210
11211 fn cache_accepted_order(
11212 cache: &Rc<RefCell<Cache>>,
11213 order: OrderAny,
11214 venue_order_id: VenueOrderId,
11215 client_id: Option<ClientId>,
11216 ) -> (InstrumentId, ClientOrderId) {
11217 let instrument_id = order.instrument_id();
11218 let client_order_id = order.client_order_id();
11219 cache
11220 .borrow_mut()
11221 .add_order(order, None, client_id, false)
11222 .unwrap();
11223
11224 let accepted = OrderEventAny::Accepted(OrderAccepted::new(
11225 trader_id(),
11226 strategy_id(),
11227 instrument_id,
11228 client_order_id,
11229 venue_order_id,
11230 account_id(),
11231 UUID4::new(),
11232 UnixNanos::default(),
11233 UnixNanos::default(),
11234 false,
11235 ));
11236 cache.borrow_mut().update_order(&accepted).unwrap();
11237
11238 (instrument_id, client_order_id)
11239 }
11240
11241 fn cache_pending_cancel_order(
11242 cache: &Rc<RefCell<Cache>>,
11243 order: OrderAny,
11244 venue_order_id: VenueOrderId,
11245 ) {
11246 let (instrument_id, client_order_id) =
11247 cache_accepted_order(cache, order, venue_order_id, Some(client_id()));
11248
11249 let pending_cancel = OrderEventAny::PendingCancel(OrderPendingCancel::new(
11250 trader_id(),
11251 strategy_id(),
11252 instrument_id,
11253 client_order_id,
11254 Some(account_id()),
11255 UUID4::new(),
11256 UnixNanos::default(),
11257 UnixNanos::default(),
11258 false,
11259 Some(venue_order_id),
11260 ));
11261 cache.borrow_mut().update_order(&pending_cancel).unwrap();
11262 }
11263
11264 fn submit_order_list_command(orders: &[OrderAny], order_list_id: &str) -> SubmitOrderList {
11265 let order_list = OrderList::new(
11266 OrderListId::from(order_list_id),
11267 orders[0].instrument_id(),
11268 strategy_id(),
11269 orders.iter().map(|order| order.client_order_id()).collect(),
11270 UnixNanos::default(),
11271 );
11272 let order_inits = orders
11273 .iter()
11274 .map(|order| order.init_event().clone())
11275 .collect();
11276
11277 SubmitOrderList::new(
11278 trader_id(),
11279 Some(client_id()),
11280 strategy_id(),
11281 order_list,
11282 order_inits,
11283 None,
11284 None,
11285 None,
11286 UUID4::new(),
11287 UnixNanos::default(),
11288 None,
11289 )
11290 }
11291
11292 fn test_contingent_limit_order(
11293 instrument_id: InstrumentId,
11294 client_order_id: &str,
11295 order_list_id: &str,
11296 linked_order_id: &str,
11297 ) -> OrderAny {
11298 OrderAny::Limit(LimitOrder::new(
11299 trader_id(),
11300 strategy_id(),
11301 instrument_id,
11302 ClientOrderId::from(client_order_id),
11303 OrderSide::Buy,
11304 Quantity::from("0.1000"),
11305 Price::from("2361.31"),
11306 TimeInForce::Gtc,
11307 None,
11308 false,
11309 false,
11310 false,
11311 None,
11312 None,
11313 None,
11314 Some(ContingencyType::Oco),
11315 Some(OrderListId::from(order_list_id)),
11316 Some(vec![ClientOrderId::from(linked_order_id)]),
11317 None,
11318 None,
11319 None,
11320 None,
11321 None,
11322 UUID4::new(),
11323 UnixNanos::default(),
11324 ))
11325 }
11326
11327 async fn recv_order_event(
11328 rx: &mut tokio::sync::mpsc::UnboundedReceiver<ExecutionEvent>,
11329 ) -> OrderEventAny {
11330 let event = tokio::time::timeout(Duration::from_secs(2), rx.recv())
11331 .await
11332 .expect("timed out waiting for execution event")
11333 .expect("execution event channel closed");
11334
11335 match event {
11336 ExecutionEvent::Order(event) => event,
11337 event => panic!("expected order event, was {event:?}"),
11338 }
11339 }
11340
11341 async fn assert_modify_rejected_reason(
11342 rx: &mut tokio::sync::mpsc::UnboundedReceiver<ExecutionEvent>,
11343 reason_part: &str,
11344 ) {
11345 match recv_order_event(rx).await {
11346 OrderEventAny::ModifyRejected(event) => {
11347 assert!(
11348 event.reason.contains(reason_part),
11349 "expected modify rejection containing `{reason_part}`, was `{}`",
11350 event.reason,
11351 );
11352 }
11353 event => panic!("expected modify rejected event, was {event:?}"),
11354 }
11355 }
11356
11357 fn assert_nonce_reusable(dispatch: &WsDispatchState) {
11358 assert_eq!(
11359 dispatch
11360 .nonce_manager
11361 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
11362 Some(TEST_NEXT_NONCE - 1),
11363 );
11364 assert_eq!(
11365 dispatch
11366 .nonce_manager
11367 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
11368 .unwrap(),
11369 TEST_NEXT_NONCE,
11370 );
11371 }
11372
11373 #[tokio::test]
11374 async fn auth_token_rotation_retries_failed_mint() {
11375 let credential = test_credential();
11376 let channels = auth_token_rotation_channels(TEST_ACCOUNT_INDEX_I64);
11377 let cancellation_token = CancellationToken::new();
11378 let mint_attempts = Arc::new(AtomicUsize::new(0));
11379 let subscribe_attempts = Arc::new(AtomicUsize::new(0));
11380
11381 let outcome = tokio::time::timeout(
11382 Duration::from_secs(3),
11383 refresh_auth_token_until_rotated(
11384 &credential,
11385 &channels,
11386 &cancellation_token,
11387 AuthTokenRefreshBackoff {
11388 initial_delay: Duration::from_millis(10),
11389 max_delay: Duration::from_millis(20),
11390 window: Duration::from_secs(1),
11391 },
11392 {
11393 let mint_attempts = Arc::clone(&mint_attempts);
11394 move |_| {
11395 let attempt = mint_attempts.fetch_add(1, Ordering::AcqRel);
11396 if attempt == 0 {
11397 Err(anyhow::anyhow!("mint unavailable"))
11398 } else {
11399 Ok(format!("token-{attempt}").into())
11400 }
11401 }
11402 },
11403 {
11404 let subscribe_attempts = Arc::clone(&subscribe_attempts);
11405 move |_channel, _token| {
11406 let subscribe_attempts = Arc::clone(&subscribe_attempts);
11407 async move {
11408 subscribe_attempts.fetch_add(1, Ordering::AcqRel);
11409 Ok::<(), crate::websocket::error::LighterWsError>(())
11410 }
11411 }
11412 },
11413 ),
11414 )
11415 .await
11416 .expect("rotation retry must complete within the test window");
11417
11418 assert_eq!(outcome, AuthTokenRefreshOutcome::Rotated);
11419 assert_eq!(
11420 mint_attempts.load(Ordering::Acquire),
11421 2,
11422 "failed mint must retry before the next refresh interval",
11423 );
11424 assert_eq!(
11425 subscribe_attempts.load(Ordering::Acquire),
11426 channels.len(),
11427 "subscriptions must wait until token mint succeeds",
11428 );
11429 }
11430
11431 #[tokio::test]
11432 async fn auth_token_rotation_retries_failed_resubscribe() {
11433 let credential = test_credential();
11434 let channels = auth_token_rotation_channels(TEST_ACCOUNT_INDEX_I64);
11435 let cancellation_token = CancellationToken::new();
11436 let mint_attempts = Arc::new(AtomicUsize::new(0));
11437 let subscribe_attempts = Arc::new(AtomicUsize::new(0));
11438
11439 let outcome = tokio::time::timeout(
11440 Duration::from_secs(3),
11441 refresh_auth_token_until_rotated(
11442 &credential,
11443 &channels,
11444 &cancellation_token,
11445 AuthTokenRefreshBackoff {
11446 initial_delay: Duration::from_millis(10),
11447 max_delay: Duration::from_millis(20),
11448 window: Duration::from_secs(1),
11449 },
11450 {
11451 let mint_attempts = Arc::clone(&mint_attempts);
11452 move |_| {
11453 let attempt = mint_attempts.fetch_add(1, Ordering::AcqRel);
11454 Ok(format!("token-{attempt}").into())
11455 }
11456 },
11457 {
11458 let subscribe_attempts = Arc::clone(&subscribe_attempts);
11459 move |_channel, _token| {
11460 let subscribe_attempts = Arc::clone(&subscribe_attempts);
11461 async move {
11462 let attempt = subscribe_attempts.fetch_add(1, Ordering::AcqRel);
11463 if attempt == 0 {
11464 Err(crate::websocket::error::LighterWsError::Client(
11465 "handler unavailable".to_string(),
11466 ))
11467 } else {
11468 Ok(())
11469 }
11470 }
11471 }
11472 },
11473 ),
11474 )
11475 .await
11476 .expect("rotation retry must complete within the test window");
11477
11478 assert_eq!(outcome, AuthTokenRefreshOutcome::Rotated);
11479 assert_eq!(
11480 mint_attempts.load(Ordering::Acquire),
11481 2,
11482 "failed resubscribe must trigger a fresh auth-token mint",
11483 );
11484 assert_eq!(
11485 subscribe_attempts.load(Ordering::Acquire),
11486 channels.len() * 2,
11487 "failed resubscribe must retry the private account-channel set",
11488 );
11489 }
11490
11491 #[tokio::test]
11492 async fn auth_token_rotation_exhausts_retry_window() {
11493 let credential = test_credential();
11494 let channels = auth_token_rotation_channels(TEST_ACCOUNT_INDEX_I64);
11495 let cancellation_token = CancellationToken::new();
11496 let mint_attempts = Arc::new(AtomicUsize::new(0));
11497
11498 let refresh = refresh_auth_token_until_rotated(
11499 &credential,
11500 &channels,
11501 &cancellation_token,
11502 AuthTokenRefreshBackoff {
11503 initial_delay: Duration::from_millis(100),
11504 max_delay: Duration::from_millis(100),
11505 window: Duration::from_secs(1),
11506 },
11507 {
11508 let mint_attempts = Arc::clone(&mint_attempts);
11509 move |_| {
11510 mint_attempts.fetch_add(1, Ordering::AcqRel);
11511 Err(anyhow::anyhow!("mint unavailable"))
11512 }
11513 },
11514 |_channel, _token| async { Ok::<(), crate::websocket::error::LighterWsError>(()) },
11515 );
11516 let observe_retry = wait_until_async(
11517 || async { mint_attempts.load(Ordering::Acquire) > 1 },
11518 Duration::from_secs(2),
11519 );
11520
11521 let (outcome, ()) = tokio::time::timeout(Duration::from_secs(3), async {
11522 tokio::join!(refresh, observe_retry)
11523 })
11524 .await
11525 .expect("rotation retry exhaustion must complete within the test window");
11526
11527 assert_eq!(outcome, AuthTokenRefreshOutcome::Exhausted);
11528 assert!(
11529 mint_attempts.load(Ordering::Acquire) > 1,
11530 "persistent mint failure must retry until the window is exhausted",
11531 );
11532 }
11533
11534 #[tokio::test]
11535 async fn auth_token_rotation_cancels_during_retry_backoff() {
11536 let credential = test_credential();
11537 let channels = auth_token_rotation_channels(TEST_ACCOUNT_INDEX_I64);
11538 let cancellation_token = CancellationToken::new();
11539 let mint_attempts = Arc::new(AtomicUsize::new(0));
11540
11541 let cancel = cancellation_token.clone();
11542 let cancel_after_first_attempt = wait_until_async(
11543 || async { mint_attempts.load(Ordering::Acquire) > 0 },
11544 Duration::from_secs(2),
11545 );
11546
11547 let refresh = refresh_auth_token_until_rotated(
11548 &credential,
11549 &channels,
11550 &cancellation_token,
11551 AuthTokenRefreshBackoff {
11552 initial_delay: Duration::from_secs(2),
11553 max_delay: Duration::from_secs(2),
11554 window: Duration::from_secs(5),
11555 },
11556 {
11557 let mint_attempts = Arc::clone(&mint_attempts);
11558 move |_| {
11559 mint_attempts.fetch_add(1, Ordering::AcqRel);
11560 Err(anyhow::anyhow!("mint unavailable"))
11561 }
11562 },
11563 |_channel, _token| async { Ok::<(), crate::websocket::error::LighterWsError>(()) },
11564 );
11565
11566 let cancel_task = async move {
11567 cancel_after_first_attempt.await;
11568 cancel.cancel();
11569 };
11570
11571 let (outcome, ()) = tokio::time::timeout(Duration::from_secs(6), async {
11572 tokio::join!(refresh, cancel_task)
11573 })
11574 .await
11575 .expect("rotation cancellation must complete within the test window");
11576
11577 assert_eq!(outcome, AuthTokenRefreshOutcome::Cancelled);
11578 assert_eq!(
11579 mint_attempts.load(Ordering::Acquire),
11580 1,
11581 "cancellation during backoff must stop before the next retry",
11582 );
11583 }
11584
11585 #[rstest]
11586 #[case::rotated(AuthTokenRefreshOutcome::Rotated, Some(AUTH_TOKEN_REFRESH_INTERVAL))]
11587 #[case::cancelled(AuthTokenRefreshOutcome::Cancelled, None)]
11588 #[case::exhausted(
11589 AuthTokenRefreshOutcome::Exhausted,
11590 Some(AUTH_TOKEN_REFRESH_RETRY_MAX_DELAY)
11591 )]
11592 fn auth_token_refresh_next_delay_matches_outcome(
11593 #[case] outcome: AuthTokenRefreshOutcome,
11594 #[case] expected: Option<Duration>,
11595 ) {
11596 assert_eq!(auth_token_refresh_next_delay(outcome), expected);
11597 }
11598
11599 #[rstest]
11600 fn auth_token_rotation_channels_match_private_account_streams() {
11601 assert_eq!(
11602 auth_token_rotation_channels(TEST_ACCOUNT_INDEX_I64),
11603 [
11604 LighterWsChannel::AccountAllOrders(TEST_ACCOUNT_INDEX_I64),
11605 LighterWsChannel::AccountAllTrades(TEST_ACCOUNT_INDEX_I64),
11606 LighterWsChannel::AccountAllPositions(TEST_ACCOUNT_INDEX_I64),
11607 LighterWsChannel::AccountAllAssets(TEST_ACCOUNT_INDEX_I64),
11608 LighterWsChannel::UserStats(TEST_ACCOUNT_INDEX_I64),
11609 ],
11610 );
11611 }
11612
11613 #[rstest]
11614 #[case::below_max(
11615 Duration::from_millis(10),
11616 Duration::from_millis(100),
11617 Duration::from_millis(20)
11618 )]
11619 #[case::at_max(
11620 Duration::from_millis(100),
11621 Duration::from_millis(100),
11622 Duration::from_millis(100)
11623 )]
11624 #[case::overflow_clamps(Duration::MAX, Duration::from_secs(300), Duration::from_secs(300))]
11625 fn next_auth_token_refresh_retry_delay_doubles_and_caps(
11626 #[case] current: Duration,
11627 #[case] max: Duration,
11628 #[case] expected: Duration,
11629 ) {
11630 assert_eq!(next_auth_token_refresh_retry_delay(current, max), expected);
11631 }
11632
11633 #[rstest]
11634 #[tokio::test]
11635 async fn auth_token_refresh_is_retained_until_session_shutdown() {
11636 let (mut client, _cache, _rx) = create_execution_client();
11637
11638 client
11639 .spawn_auth_token_refresh(test_credential())
11640 .expect("spawn auth token refresh");
11641 client
11642 .nonce_recovery_inflight
11643 .store(true, Ordering::Release);
11644
11645 assert!(
11646 client
11647 .auth_refresh_handle
11648 .as_ref()
11649 .is_some_and(|handle| !handle.is_finished()),
11650 );
11651
11652 client.begin_session_shutdown();
11653 client
11654 .finish_session_shutdown()
11655 .await
11656 .expect("session shutdown");
11657
11658 assert!(client.auth_refresh_handle.is_none());
11659 assert!(!client.nonce_recovery_inflight.load(Ordering::Acquire));
11660 }
11661
11662 #[rstest]
11663 #[tokio::test]
11664 async fn session_shutdown_rejects_tasks_registered_during_abort() {
11665 struct SpawnOnDrop {
11666 pending_tasks: TaskSpawner,
11667 rejected: Arc<AtomicBool>,
11668 polled: Arc<AtomicBool>,
11669 }
11670
11671 impl Drop for SpawnOnDrop {
11672 fn drop(&mut self) {
11673 let polled = Arc::clone(&self.polled);
11674 let result = self.pending_tasks.spawn(async move {
11675 polled.store(true, Ordering::Release);
11676 });
11677 self.rejected.store(result.is_err(), Ordering::Release);
11678 }
11679 }
11680
11681 let (mut client, _cache, _rx) = create_execution_client();
11682 let rejected = Arc::new(AtomicBool::new(false));
11683 let polled = Arc::new(AtomicBool::new(false));
11684 let pending_spawner = client.pending_tasks.spawner().expect("task spawner");
11685 let guard = SpawnOnDrop {
11686 pending_tasks: pending_spawner.clone(),
11687 rejected: Arc::clone(&rejected),
11688 polled: Arc::clone(&polled),
11689 };
11690
11691 pending_spawner
11692 .spawn(async move {
11693 let _guard = guard;
11694 std::future::pending::<()>().await;
11695 })
11696 .expect("parent task spawn");
11697
11698 client
11699 .finish_session_shutdown()
11700 .await
11701 .expect("session shutdown");
11702
11703 assert!(client.pending_tasks.is_empty());
11704 assert!(rejected.load(Ordering::Acquire));
11705 assert!(!polled.load(Ordering::Acquire));
11706 }
11707
11708 #[rstest]
11709 #[tokio::test]
11710 async fn session_shutdown_discards_pending_sendtx_before_epoch_reuse() {
11711 struct EnqueueOnDrop {
11712 dispatch: WsDispatchState,
11713 pending: Option<PendingSendTx>,
11714 }
11715
11716 impl Drop for EnqueueOnDrop {
11717 fn drop(&mut self) {
11718 self.dispatch
11719 .enqueue_pending_sendtx(self.pending.take().expect("pending sendTx"));
11720 }
11721 }
11722
11723 let (mut client, _cache, _rx) = create_execution_client();
11724 let old_epoch = client.ws_client.connection_epoch();
11725 client.dispatch.enqueue_pending_sendtx(PendingSendTx {
11726 batch_id: None,
11727 connection_epoch: old_epoch,
11728 kind: PendingSendTxKind::Other,
11729 submitted_at: UnixNanos::default(),
11730 nonce: TEST_NEXT_NONCE,
11731 api_key_index: TEST_API_KEY_INDEX,
11732 tx_hash: "shutdown-pending".to_string(),
11733 });
11734 let guard = EnqueueOnDrop {
11735 dispatch: client.dispatch.clone(),
11736 pending: Some(PendingSendTx {
11737 batch_id: None,
11738 connection_epoch: old_epoch,
11739 kind: PendingSendTxKind::Other,
11740 submitted_at: UnixNanos::default(),
11741 nonce: TEST_NEXT_NONCE + 1,
11742 api_key_index: TEST_API_KEY_INDEX,
11743 tx_hash: "shutdown-late".to_string(),
11744 }),
11745 };
11746
11747 client
11748 .pending_tasks
11749 .spawn(async move {
11750 let _guard = guard;
11751 std::future::pending::<()>().await;
11752 })
11753 .expect("pending sendTx producer spawn");
11754 client
11755 .ws_stream_handle
11756 .insert(get_runtime().spawn(std::future::pending()));
11757
11758 client.begin_session_shutdown();
11759 client
11760 .finish_session_shutdown()
11761 .await
11762 .expect("session shutdown");
11763
11764 assert_eq!(client.ws_client.connection_epoch(), old_epoch);
11765 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
11766 }
11767
11768 #[rstest]
11769 fn replacing_ws_client_preserves_cached_instruments() {
11770 let (mut client, cache, _rx) = create_execution_client();
11771 let instrument_id = register_test_instrument(&client, &cache);
11772 let instrument = cache
11773 .borrow()
11774 .instrument(&instrument_id)
11775 .expect("instrument")
11776 .clone();
11777 client
11778 .ws_client
11779 .cache_instrument(TEST_MARKET_INDEX, instrument.clone());
11780
11781 let _old_ws_client = client.take_ws_client();
11782
11783 let cached_instruments = client.ws_client.instruments_cache();
11784 assert_eq!(
11785 cached_instruments
11786 .get(&TEST_MARKET_INDEX)
11787 .expect("cached instrument")
11788 .value(),
11789 &instrument,
11790 );
11791 }
11792
11793 #[rstest]
11794 fn tx_send_sequencer_allocates_under_registration_lock() {
11795 let sequencer = TxSendSequencer::new();
11796
11797 let mut lower = sequencer
11798 .reserve(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX, || {
11799 assert!(
11800 sequencer.state.try_lock().is_none(),
11801 "allocation must be serialized with registration"
11802 );
11803 Ok(TEST_NEXT_NONCE)
11804 })
11805 .unwrap();
11806
11807 let higher = sequencer
11808 .reserve(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX, || {
11809 Ok(TEST_NEXT_NONCE + 1)
11810 })
11811 .unwrap();
11812
11813 assert!(!sequencer.ready_to_send(higher.key, higher.nonce));
11814 lower.release();
11815 assert!(sequencer.ready_to_send(higher.key, higher.nonce));
11816 }
11817
11818 #[rstest]
11819 fn tx_send_sequencer_retains_reused_nonce_reservation() {
11820 let (client, _cache, _rx) = create_execution_client();
11821 let context = client
11822 .fanout_dispatch_context(client.credential.as_ref().unwrap())
11823 .unwrap();
11824 let mut original = context.build_tx_context().unwrap();
11825 let nonce = original.context.nonce;
11826 rollback_tx_dispatch(
11827 &client.dispatch,
11828 client.credential.as_ref().unwrap(),
11829 None,
11830 nonce,
11831 );
11832 let mut replacement = context.build_tx_context().unwrap();
11833 let higher = context.build_tx_context().unwrap();
11834 original.send_reservation.release();
11835 assert_eq!(replacement.context.nonce, nonce);
11836 assert_eq!(higher.context.nonce, nonce + 1);
11837 assert!(
11838 !context
11839 .tx_send_sequencer
11840 .ready_to_send(higher.send_reservation.key, higher.context.nonce)
11841 );
11842 replacement.send_reservation.release();
11843 assert!(
11844 context
11845 .tx_send_sequencer
11846 .ready_to_send(higher.send_reservation.key, higher.context.nonce)
11847 );
11848 }
11849
11850 #[tokio::test]
11851 async fn tx_send_sequencer_blocks_higher_nonce_until_lower_batch_releases() {
11852 let sequencer = TxSendSequencer::new();
11853
11854 let mut lower_a = sequencer
11855 .reserve(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX, || {
11856 Ok(TEST_NEXT_NONCE)
11857 })
11858 .unwrap();
11859
11860 let mut lower_b = sequencer
11861 .reserve(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX, || {
11862 Ok(TEST_NEXT_NONCE + 1)
11863 })
11864 .unwrap();
11865
11866 let mut higher = sequencer
11867 .reserve(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX, || {
11868 Ok(TEST_NEXT_NONCE + 2)
11869 })
11870 .unwrap();
11871
11872 let (sent_tx, mut sent_rx) = tokio::sync::mpsc::unbounded_channel();
11873
11874 let higher_task = tokio::spawn(async move {
11875 higher.wait_for_turn().await;
11876 sent_tx.send(higher.nonce).unwrap();
11877 higher.release();
11878 });
11879
11880 assert!(
11881 tokio::time::timeout(Duration::from_millis(50), sent_rx.recv())
11882 .await
11883 .is_err(),
11884 "higher nonce must wait while the lower batch is pending",
11885 );
11886
11887 {
11888 let lower_reservations = [&lower_a, &lower_b];
11889 tokio::time::timeout(
11890 Duration::from_millis(50),
11891 wait_for_tx_send_reservations(&lower_reservations),
11892 )
11893 .await
11894 .expect("lower batch must already have the send turn");
11895 }
11896
11897 lower_a.release();
11898 lower_b.release();
11899
11900 let sent = tokio::time::timeout(Duration::from_secs(2), sent_rx.recv())
11901 .await
11902 .expect("higher nonce must proceed after lower batch releases")
11903 .expect("send channel must stay open");
11904 higher_task.await.unwrap();
11905
11906 assert_eq!(
11907 sent,
11908 TEST_NEXT_NONCE + 2,
11909 "higher nonce must send after the lower batch releases",
11910 );
11911 }
11912
11913 #[tokio::test]
11914 async fn tx_send_reservation_drop_releases_lower_nonce() {
11915 let sequencer = TxSendSequencer::new();
11916
11917 let lower = sequencer
11918 .reserve(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX, || {
11919 Ok(TEST_NEXT_NONCE)
11920 })
11921 .unwrap();
11922
11923 let mut higher = sequencer
11924 .reserve(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX, || {
11925 Ok(TEST_NEXT_NONCE + 1)
11926 })
11927 .unwrap();
11928
11929 let (sent_tx, mut sent_rx) = tokio::sync::mpsc::unbounded_channel();
11930
11931 let higher_task = tokio::spawn(async move {
11932 higher.wait_for_turn().await;
11933 sent_tx.send(higher.nonce).unwrap();
11934 higher.release();
11935 });
11936
11937 assert!(
11938 tokio::time::timeout(Duration::from_millis(50), sent_rx.recv())
11939 .await
11940 .is_err(),
11941 "higher nonce must wait while the lower reservation is pending",
11942 );
11943
11944 drop(lower);
11945
11946 let sent = tokio::time::timeout(Duration::from_secs(2), sent_rx.recv())
11947 .await
11948 .expect("higher nonce must proceed after lower reservation drops")
11949 .expect("send channel must stay open");
11950 higher_task.await.unwrap();
11951
11952 assert_eq!(
11953 sent,
11954 TEST_NEXT_NONCE + 1,
11955 "higher nonce must send after the lower reservation drops",
11956 );
11957 }
11958
11959 #[tokio::test]
11960 async fn tx_send_sequencer_keeps_nonce_streams_independent() {
11961 let sequencer = TxSendSequencer::new();
11962
11963 let _other_account = sequencer
11964 .reserve(TEST_ACCOUNT_INDEX_I64 + 1, TEST_API_KEY_INDEX, || {
11965 Ok(TEST_NEXT_NONCE)
11966 })
11967 .unwrap();
11968
11969 let _other_api_key = sequencer
11970 .reserve(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX + 1, || {
11971 Ok(TEST_NEXT_NONCE)
11972 })
11973 .unwrap();
11974
11975 let mut current = sequencer
11976 .reserve(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX, || {
11977 Ok(TEST_NEXT_NONCE + 10)
11978 })
11979 .unwrap();
11980
11981 tokio::time::timeout(Duration::from_millis(50), current.wait_for_turn())
11982 .await
11983 .expect("lower nonces for other keys must not block this key");
11984 current.release();
11985 }
11986
11987 #[rstest]
11988 fn tx_dispatch_guard_rolls_back_nonce_and_cloid_when_armed() {
11989 let dispatch = WsDispatchState::new();
11990 let credential = test_credential();
11991 dispatch
11992 .nonce_manager
11993 .refresh(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX, TEST_NEXT_NONCE);
11994
11995 let cloid = ClientOrderId::from("O-GUARD-ARMED");
11996 let client_order_index = dispatch.derive_client_order_index(&cloid);
11997 dispatch.register_cloid(client_order_index, cloid).unwrap();
11998 let nonce = dispatch
11999 .nonce_manager
12000 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
12001 .unwrap();
12002
12003 {
12004 let _guard = TxDispatchGuard::new(
12005 dispatch.clone(),
12006 &credential,
12007 Some(client_order_index),
12008 nonce,
12009 );
12010 }
12011
12012 assert_nonce_reusable(&dispatch);
12013 assert!(dispatch.cloid_map.get(&client_order_index).is_none());
12014 }
12015
12016 #[rstest]
12017 fn tx_dispatch_guard_rolls_back_nonce_without_cloid_when_armed() {
12018 let dispatch = WsDispatchState::new();
12019 let credential = test_credential();
12020 dispatch
12021 .nonce_manager
12022 .refresh(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX, TEST_NEXT_NONCE);
12023 let nonce = dispatch
12024 .nonce_manager
12025 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
12026 .unwrap();
12027
12028 {
12029 let _guard = TxDispatchGuard::new(dispatch.clone(), &credential, None, nonce);
12030 }
12031
12032 assert_nonce_reusable(&dispatch);
12033 assert!(dispatch.cloid_map.is_empty());
12034 }
12035
12036 #[rstest]
12037 fn tx_dispatch_guard_preserves_nonce_and_cloid_when_disarmed() {
12038 let dispatch = WsDispatchState::new();
12039 let credential = test_credential();
12040 dispatch
12041 .nonce_manager
12042 .refresh(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX, TEST_NEXT_NONCE);
12043
12044 let cloid = ClientOrderId::from("O-GUARD-DISARMED");
12045 let client_order_index = dispatch.derive_client_order_index(&cloid);
12046 dispatch.register_cloid(client_order_index, cloid).unwrap();
12047 let nonce = dispatch
12048 .nonce_manager
12049 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
12050 .unwrap();
12051
12052 {
12053 let mut guard = TxDispatchGuard::new(
12054 dispatch.clone(),
12055 &credential,
12056 Some(client_order_index),
12057 nonce,
12058 );
12059 guard.disarm();
12060 }
12061
12062 assert_eq!(
12063 dispatch
12064 .nonce_manager
12065 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
12066 Some(TEST_NEXT_NONCE),
12067 );
12068 assert_eq!(
12069 dispatch
12070 .cloid_map
12071 .get(&client_order_index)
12072 .map(|entry| *entry.value()),
12073 Some(cloid),
12074 );
12075 }
12076
12077 #[tokio::test]
12078 async fn submit_order_send_failure_emits_submitted_then_rejected_and_rolls_back() {
12079 let (client, cache, mut rx) = create_execution_client();
12080 let instrument_id = register_test_instrument(&client, &cache);
12081 let mut factory = test_order_factory();
12082 let order = test_limit_order(&mut factory, instrument_id, "O-SUBMIT-FAIL");
12083 let client_order_index = client
12084 .dispatch
12085 .derive_client_order_index(&order.client_order_id());
12086 cache_order(&cache, order.clone());
12087
12088 let command = SubmitOrder::from_order(
12089 &order,
12090 trader_id(),
12091 Some(client_id()),
12092 None,
12093 UUID4::new(),
12094 UnixNanos::default(),
12095 );
12096 client.submit_order(command).unwrap();
12097
12098 let submitted = recv_order_event(&mut rx).await;
12099 let rejected = recv_order_event(&mut rx).await;
12100
12101 match submitted {
12102 OrderEventAny::Submitted(event) => {
12103 assert_eq!(event.client_order_id, order.client_order_id());
12104 assert_eq!(event.instrument_id, instrument_id);
12105 }
12106 event => panic!("expected submitted event, was {event:?}"),
12107 }
12108
12109 match rejected {
12110 OrderEventAny::Rejected(event) => {
12111 assert_eq!(event.client_order_id, order.client_order_id());
12112 assert_eq!(event.instrument_id, instrument_id);
12113 assert!(
12114 event
12115 .reason
12116 .contains("Lighter submit_order dispatch failed"),
12117 );
12118 assert!(event.reason.contains("handler unavailable"));
12119 }
12120 event => panic!("expected rejected event, was {event:?}"),
12121 }
12122
12123 assert!(client.dispatch.cloid_map.get(&client_order_index).is_none());
12124 assert_nonce_reusable(&client.dispatch);
12125 assert_eq!(
12126 client.dispatch.pending_sendtx_len(),
12127 0,
12128 "local-send-failure must remove the pending entry by nonce",
12129 );
12130 }
12131
12132 #[tokio::test]
12133 async fn submit_order_unknown_send_outcome_retains_pending_state() {
12134 let (client, cache, mut rx) = create_execution_client();
12135 let instrument_id = register_test_instrument(&client, &cache);
12136 let mut factory = test_order_factory();
12137 let order = test_limit_order(&mut factory, instrument_id, "O-SUBMIT-UNKNOWN");
12138 let client_order_index = client
12139 .dispatch
12140 .derive_client_order_index(&order.client_order_id());
12141 cache_order(&cache, order.clone());
12142 client.ws_client.drop_next_send_tx_result_for_test().await;
12143
12144 client
12145 .submit_order(SubmitOrder::from_order(
12146 &order,
12147 trader_id(),
12148 Some(client_id()),
12149 None,
12150 UUID4::new(),
12151 UnixNanos::default(),
12152 ))
12153 .unwrap();
12154
12155 assert!(matches!(
12156 recv_order_event(&mut rx).await,
12157 OrderEventAny::Submitted(_)
12158 ));
12159 assert!(
12160 tokio::time::timeout(Duration::from_millis(50), rx.recv())
12161 .await
12162 .is_err(),
12163 "unknown outcome must not emit OrderRejected",
12164 );
12165 assert!(client.dispatch.cloid_map.contains_key(&client_order_index));
12166 assert_eq!(client.dispatch.pending_sendtx_len(), 1);
12167 assert_eq!(
12168 client
12169 .dispatch
12170 .nonce_manager
12171 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
12172 Some(TEST_NEXT_NONCE),
12173 );
12174
12175 let stale = client.dispatch.drain_pending_sendtx(0);
12176 assert_eq!(stale.len(), 1);
12177 assert!(matches!(stale[0].kind, PendingSendTxKind::Create { .. }));
12178 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
12179 assert!(client.dispatch.cloid_map.contains_key(&client_order_index));
12180 assert!(
12181 client
12182 .dispatch
12183 .order_identity(&order.client_order_id())
12184 .is_some(),
12185 "reconnect draining must retain identity for reconciliation",
12186 );
12187 assert_eq!(
12188 client
12189 .dispatch
12190 .nonce_manager
12191 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
12192 Some(TEST_NEXT_NONCE),
12193 );
12194 }
12195
12196 #[tokio::test]
12197 async fn submit_order_cloid_probe_exhaustion_denies_without_overwrite() {
12198 let (client, cache, mut rx) = create_execution_client();
12199 let instrument_id = register_test_instrument(&client, &cache);
12200 let mut factory = test_order_factory();
12201 let order = test_limit_order(&mut factory, instrument_id, "O-PROBE-EXHAUSTED");
12202 let cloid = order.client_order_id();
12203 let mut index = client.dispatch.derive_client_order_index(&cloid);
12204 let mut existing = Vec::new();
12205
12206 for attempt in 0..=crate::websocket::dispatch::CLOID_INDEX_PROBE_LIMIT {
12207 let existing_cloid = ClientOrderId::from(format!("EXISTING-{attempt}"));
12208 client.dispatch.cloid_map.insert(index, existing_cloid);
12209 existing.push((index, existing_cloid));
12210 index = if index == i64::from(i32::MAX) {
12211 0
12212 } else {
12213 index + 1
12214 };
12215 }
12216
12217 cache_order(&cache, order.clone());
12218
12219 client
12220 .submit_order(SubmitOrder::from_order(
12221 &order,
12222 trader_id(),
12223 Some(client_id()),
12224 None,
12225 UUID4::new(),
12226 UnixNanos::default(),
12227 ))
12228 .unwrap();
12229
12230 let event = recv_order_event(&mut rx).await;
12231 assert!(matches!(event, OrderEventAny::Denied(_)));
12232 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
12233 assert!(client.dispatch.order_identity(&cloid).is_none());
12234
12235 for (existing_index, existing_cloid) in existing {
12236 assert_eq!(
12237 client
12238 .dispatch
12239 .cloid_map
12240 .get(&existing_index)
12241 .map(|entry| *entry.value()),
12242 Some(existing_cloid),
12243 );
12244 }
12245 }
12246
12247 #[tokio::test]
12248 async fn submit_sell_order_send_failure_dispatches_and_rolls_back() {
12249 let (client, cache, mut rx) = create_execution_client();
12252 let instrument_id = register_test_instrument(&client, &cache);
12253 let mut factory = test_order_factory();
12254 let order = test_limit_order_with(
12255 &mut factory,
12256 instrument_id,
12257 "O-SUBMIT-FAIL-SELL",
12258 OrderSide::Sell,
12259 TimeInForce::Gtc,
12260 None,
12261 false,
12262 );
12263 let client_order_index = client
12264 .dispatch
12265 .derive_client_order_index(&order.client_order_id());
12266 cache_order(&cache, order.clone());
12267
12268 let command = SubmitOrder::from_order(
12269 &order,
12270 trader_id(),
12271 Some(client_id()),
12272 None,
12273 UUID4::new(),
12274 UnixNanos::default(),
12275 );
12276 client.submit_order(command).unwrap();
12277
12278 let _submitted = recv_order_event(&mut rx).await;
12279 let rejected = recv_order_event(&mut rx).await;
12280
12281 match rejected {
12282 OrderEventAny::Rejected(event) => {
12283 assert_eq!(event.client_order_id, order.client_order_id());
12284 assert!(
12285 event
12286 .reason
12287 .contains("Lighter submit_order dispatch failed"),
12288 );
12289 }
12290 event => panic!("expected rejected event, was {event:?}"),
12291 }
12292
12293 assert!(client.dispatch.cloid_map.get(&client_order_index).is_none());
12294 assert_nonce_reusable(&client.dispatch);
12295 }
12296
12297 #[tokio::test]
12298 async fn submit_gtd_order_with_explicit_expiry_dispatches_and_rolls_back() {
12299 let (client, cache, mut rx) = create_execution_client();
12304 let instrument_id = register_test_instrument(&client, &cache);
12305 let mut factory = test_order_factory();
12306 let expiry = client.clock.get_time_ns() + DurationNanos::from_mins(10);
12307 let order = test_limit_order_with(
12308 &mut factory,
12309 instrument_id,
12310 "O-SUBMIT-FAIL-GTD",
12311 OrderSide::Buy,
12312 TimeInForce::Gtd,
12313 Some(expiry),
12314 false,
12315 );
12316 let client_order_index = client
12317 .dispatch
12318 .derive_client_order_index(&order.client_order_id());
12319 cache_order(&cache, order.clone());
12320
12321 let command = SubmitOrder::from_order(
12322 &order,
12323 trader_id(),
12324 Some(client_id()),
12325 None,
12326 UUID4::new(),
12327 UnixNanos::default(),
12328 );
12329 client.submit_order(command).unwrap();
12330
12331 let _submitted = recv_order_event(&mut rx).await;
12332 let rejected = recv_order_event(&mut rx).await;
12333
12334 match rejected {
12335 OrderEventAny::Rejected(event) => {
12336 assert!(
12337 event
12338 .reason
12339 .contains("Lighter submit_order dispatch failed"),
12340 );
12341 }
12342 event => panic!("expected rejected event, was {event:?}"),
12343 }
12344
12345 assert!(client.dispatch.cloid_map.get(&client_order_index).is_none());
12346 assert_nonce_reusable(&client.dispatch);
12347 }
12348
12349 #[tokio::test]
12350 async fn submit_gtd_order_with_short_expiry_is_denied_before_nonce_allocation() {
12351 let (client, cache, mut rx) = create_execution_client();
12352 let instrument_id = register_test_instrument(&client, &cache);
12353 let mut factory = test_order_factory();
12354 let expiry = UnixNanos::from(
12355 client
12356 .clock
12357 .get_time_ns()
12358 .as_u64()
12359 .saturating_add(60 * 1_000_000_000),
12360 );
12361 let order = test_limit_order_with(
12362 &mut factory,
12363 instrument_id,
12364 "O-SHORT-GTD",
12365 OrderSide::Buy,
12366 TimeInForce::Gtd,
12367 Some(expiry),
12368 false,
12369 );
12370 cache_order(&cache, order.clone());
12371
12372 let command = SubmitOrder::from_order(
12373 &order,
12374 trader_id(),
12375 Some(client_id()),
12376 None,
12377 UUID4::new(),
12378 UnixNanos::default(),
12379 );
12380 client.submit_order(command).unwrap();
12381
12382 let denied = recv_order_event(&mut rx).await;
12383 match denied {
12384 OrderEventAny::Denied(event) => {
12385 assert!(event.reason.contains("at least 5 minutes"));
12386 }
12387 event => panic!("expected denied event, was {event:?}"),
12388 }
12389
12390 assert_nonce_reusable(&client.dispatch);
12391 }
12392
12393 #[tokio::test]
12394 async fn submit_gtd_order_with_short_expiry_dispatches_when_use_gtd_false() {
12395 let (client, cache, mut rx) =
12400 create_execution_client_with_config(test_config_with_use_gtd(false));
12401 let instrument_id = register_test_instrument(&client, &cache);
12402 let mut factory = test_order_factory();
12403 let expiry = UnixNanos::from(
12404 client
12405 .clock
12406 .get_time_ns()
12407 .as_u64()
12408 .saturating_add(60 * 1_000_000_000),
12409 );
12410 let order = test_limit_order_with(
12411 &mut factory,
12412 instrument_id,
12413 "O-SHORT-GTD-MANAGED",
12414 OrderSide::Buy,
12415 TimeInForce::Gtd,
12416 Some(expiry),
12417 false,
12418 );
12419 let client_order_index = client
12420 .dispatch
12421 .derive_client_order_index(&order.client_order_id());
12422 cache_order(&cache, order.clone());
12423
12424 let command = SubmitOrder::from_order(
12425 &order,
12426 trader_id(),
12427 Some(client_id()),
12428 None,
12429 UUID4::new(),
12430 UnixNanos::default(),
12431 );
12432 client.submit_order(command).unwrap();
12433
12434 let _submitted = recv_order_event(&mut rx).await;
12435 let rejected = recv_order_event(&mut rx).await;
12436
12437 match rejected {
12438 OrderEventAny::Rejected(event) => {
12439 assert!(
12440 event
12441 .reason
12442 .contains("Lighter submit_order dispatch failed"),
12443 );
12444 }
12445 event => panic!("expected rejected event, was {event:?}"),
12446 }
12447
12448 assert!(client.dispatch.cloid_map.get(&client_order_index).is_none());
12449 assert_nonce_reusable(&client.dispatch);
12450 }
12451
12452 #[rstest]
12453 fn prepare_managed_gtd_plan_uses_safe_venue_expiry() {
12454 let (client, cache, _rx) =
12455 create_execution_client_with_config(test_config_with_use_gtd(false));
12456 let instrument_id = register_test_instrument(&client, &cache);
12457 let mut factory = test_order_factory();
12458 let expiry = UnixNanos::from(
12459 client
12460 .clock
12461 .get_time_ns()
12462 .as_u64()
12463 .saturating_add(60 * 1_000_000_000),
12464 );
12465 let order = test_limit_order_with(
12466 &mut factory,
12467 instrument_id,
12468 "O-SHORT-GTD-PLAN",
12469 OrderSide::Buy,
12470 TimeInForce::Gtd,
12471 Some(expiry),
12472 false,
12473 );
12474
12475 let before_ms = (client.clock.get_time_ns().as_u64() / 1_000_000) as i64;
12476 let plan = client
12477 .prepare_create_order_plan(&order, 0, None)
12478 .expect("managed GTD plan must validate");
12479 let after_ms = (client.clock.get_time_ns().as_u64() / 1_000_000) as i64;
12480
12481 assert!(
12482 (before_ms + crate::websocket::dispatch::ORDER_EXPIRY_DEFAULT_GTC_MS
12483 ..=after_ms + crate::websocket::dispatch::ORDER_EXPIRY_DEFAULT_GTC_MS)
12484 .contains(&plan.order_expiry)
12485 );
12486 assert_eq!(
12487 plan.time_in_force,
12488 crate::common::enums::LighterTimeInForce::GoodTillTime as u8,
12489 );
12490 }
12491
12492 #[tokio::test]
12493 async fn submit_reduce_only_order_dispatches_and_rolls_back() {
12494 let (client, cache, mut rx) = create_execution_client();
12496 let instrument_id = register_test_instrument(&client, &cache);
12497 let mut factory = test_order_factory();
12498 let order = test_limit_order_with(
12499 &mut factory,
12500 instrument_id,
12501 "O-SUBMIT-FAIL-REDUCE",
12502 OrderSide::Sell,
12503 TimeInForce::Gtc,
12504 None,
12505 true,
12506 );
12507 assert!(order.is_reduce_only());
12508 let client_order_index = client
12509 .dispatch
12510 .derive_client_order_index(&order.client_order_id());
12511 cache_order(&cache, order.clone());
12512
12513 let command = SubmitOrder::from_order(
12514 &order,
12515 trader_id(),
12516 Some(client_id()),
12517 None,
12518 UUID4::new(),
12519 UnixNanos::default(),
12520 );
12521 client.submit_order(command).unwrap();
12522
12523 let _submitted = recv_order_event(&mut rx).await;
12524 let rejected = recv_order_event(&mut rx).await;
12525
12526 match rejected {
12527 OrderEventAny::Rejected(event) => {
12528 assert!(
12529 event
12530 .reason
12531 .contains("Lighter submit_order dispatch failed"),
12532 );
12533 }
12534 event => panic!("expected rejected event, was {event:?}"),
12535 }
12536
12537 assert!(client.dispatch.cloid_map.get(&client_order_index).is_none());
12538 assert_nonce_reusable(&client.dispatch);
12539 }
12540
12541 #[rstest]
12542 #[case::nonce_refresh(false)]
12543 #[case::submission_gate(true)]
12544 #[tokio::test]
12545 async fn batch_signing_retries_after_nonce_readiness_recovers(#[case] lock_submission: bool) {
12546 let (client, _cache, _rx) = create_execution_client();
12547 let credential = test_credential();
12548 let context = client.fanout_dispatch_context(&credential).unwrap();
12549 let connection_epoch = context.ws_client.connection_epoch();
12550 let guard = if lock_submission {
12551 Some(context.nonce_submission_gate.write().await)
12552 } else {
12553 context
12554 .nonce_ready_connection_epoch
12555 .store(NONCE_CONNECTION_EPOCH_UNAVAILABLE, Ordering::Release);
12556 None
12557 };
12558
12559 let error = context.build_tx_context().unwrap_err();
12560 drop(guard);
12561 context
12562 .nonce_ready_connection_epoch
12563 .store(connection_epoch, Ordering::Release);
12564 let reserved = context.build_tx_context().unwrap();
12565
12566 assert_eq!(reserved.context.nonce, TEST_NEXT_NONCE);
12567 assert!(should_retry_batch_signing(&error));
12568 assert_eq!(
12569 error.to_string(),
12570 if lock_submission {
12571 "Lighter nonce refresh is in progress".to_string()
12572 } else {
12573 format!("Lighter nonce refresh is pending for connection epoch {connection_epoch}")
12574 }
12575 );
12576 }
12577
12578 #[rstest]
12579 #[case::state_changed(TxPreparationError::StateChanged.into(), true)]
12580 #[case::skip_window(NonceError::SkipWindowExhausted {
12581 account_index: TEST_ACCOUNT_INDEX_I64,
12582 api_key_index: TEST_API_KEY_INDEX,
12583 outstanding: 17,
12584 skip_window: 16,
12585 }.into(), true)]
12586 #[case::uninitialized(NonceError::NotInitialized {
12587 account_index: TEST_ACCOUNT_INDEX_I64,
12588 api_key_index: TEST_API_KEY_INDEX,
12589 }.into(), false)]
12590 #[case::signing(anyhow::anyhow!("invalid signing key"), false)]
12591 fn batch_signing_retry_classifies_preparation_errors(
12592 #[case] error: anyhow::Error,
12593 #[case] expected: bool,
12594 ) {
12595 assert_eq!(should_retry_batch_signing(&error), expected);
12596 }
12597
12598 #[rstest]
12599 #[case::submit(false)]
12600 #[case::modify(true)]
12601 #[tokio::test]
12602 async fn batch_orders_wait_for_nonce_readiness(#[case] modify: bool) {
12603 let (client, cache, mut rx) = create_execution_client();
12604 let instrument_id = register_test_instrument(&client, &cache);
12605 let mut factory = test_order_factory();
12606 let orders: Vec<_> = ["O-READY-A", "O-READY-B"]
12607 .into_iter()
12608 .map(|id| test_limit_order(&mut factory, instrument_id, id))
12609 .collect();
12610
12611 for order in &orders {
12612 cache_order(&cache, order.clone());
12613 }
12614
12615 client.ws_client.drop_next_send_tx_result_for_test().await;
12616 client
12617 .nonce_ready_connection_epoch
12618 .store(NONCE_CONNECTION_EPOCH_UNAVAILABLE, Ordering::Release);
12619
12620 if modify {
12621 let modifies = orders
12622 .iter()
12623 .enumerate()
12624 .map(|(i, order)| {
12625 ModifyOrder::new(
12626 trader_id(),
12627 Some(client_id()),
12628 strategy_id(),
12629 instrument_id,
12630 order.client_order_id(),
12631 Some(VenueOrderId::from(format!("{}", 123 + i))),
12632 Some(Quantity::from("0.2000")),
12633 Some(Price::from("2362.00")),
12634 None,
12635 UUID4::new(),
12636 UnixNanos::default(),
12637 None,
12638 None,
12639 )
12640 })
12641 .collect();
12642
12643 client
12644 .batch_modify_orders(BatchModifyOrders::new(
12645 trader_id(),
12646 Some(client_id()),
12647 strategy_id(),
12648 instrument_id,
12649 modifies,
12650 UUID4::new(),
12651 UnixNanos::default(),
12652 None,
12653 None,
12654 ))
12655 .unwrap();
12656 } else {
12657 client
12658 .submit_order_list(submit_order_list_command(&orders, "OL-READY"))
12659 .unwrap();
12660 }
12661
12662 assert!(
12663 tokio::time::timeout(Duration::from_millis(100), rx.recv())
12664 .await
12665 .is_err()
12666 );
12667 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
12668 assert_eq!(
12669 client
12670 .dispatch
12671 .nonce_manager
12672 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
12673 Some(TEST_NEXT_NONCE - 1)
12674 );
12675 client
12676 .nonce_ready_connection_epoch
12677 .store(0, Ordering::Release);
12678 wait_until_async(
12679 || async { client.dispatch.pending_sendtx_len() == 2 },
12680 Duration::from_secs(2),
12681 )
12682 .await;
12683
12684 let pending = client.dispatch.pending_sendtx.lock().clone();
12685 assert_eq!(pending.len(), 2);
12686 assert_eq!(pending[0].batch_id, pending[1].batch_id);
12687 assert!(
12688 pending[0]
12689 .batch_id
12690 .as_ref()
12691 .unwrap()
12692 .starts_with(TX_BATCH_ID_PREFIX)
12693 );
12694
12695 for (i, (pending, order)) in pending.iter().zip(&orders).enumerate() {
12696 assert_eq!(pending.nonce, TEST_NEXT_NONCE + i as i64);
12697 assert_eq!(pending.api_key_index, TEST_API_KEY_INDEX);
12698 assert_eq!(pending.connection_epoch, 0);
12699
12700 if modify {
12701 assert!(
12702 matches!(&pending.kind, PendingSendTxKind::Modify { client_order_id, .. } if *client_order_id == order.client_order_id())
12703 );
12704 assert_eq!(
12705 client
12706 .dispatch
12707 .pending_order_action(&order.client_order_id()),
12708 Some(PendingOrderAction::Modify)
12709 );
12710 } else {
12711 assert!(
12712 matches!(&pending.kind, PendingSendTxKind::Create { order: pending_order, .. } if pending_order.client_order_id() == order.client_order_id())
12713 );
12714 let event = recv_order_event(&mut rx).await;
12715
12716 let OrderEventAny::Submitted(event) = event else {
12717 panic!("expected Submitted after nonce readiness")
12718 };
12719
12720 assert_eq!(event.client_order_id, order.client_order_id());
12721 }
12722 }
12723
12724 assert!(
12725 tokio::time::timeout(Duration::from_millis(50), rx.recv())
12726 .await
12727 .is_err()
12728 );
12729 assert_eq!(
12730 client
12731 .dispatch
12732 .nonce_manager
12733 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
12734 Some(TEST_NEXT_NONCE + 1)
12735 );
12736 }
12737
12738 #[tokio::test]
12739 async fn submit_order_list_send_failure_emits_submitted_then_rejected_and_rolls_back() {
12740 let (client, cache, mut rx) = create_execution_client();
12741 let instrument_id = register_test_instrument(&client, &cache);
12742 let mut factory = test_order_factory();
12743 let order_a = test_limit_order(&mut factory, instrument_id, "O-LIST-FAIL-A");
12744 let order_b = test_limit_order(&mut factory, instrument_id, "O-LIST-FAIL-B");
12745 let index_a = client
12746 .dispatch
12747 .derive_client_order_index(&order_a.client_order_id());
12748 let index_b = client
12749 .dispatch
12750 .derive_client_order_index(&order_b.client_order_id());
12751 cache_order(&cache, order_a.clone());
12752 cache_order(&cache, order_b.clone());
12753
12754 let command = submit_order_list_command(&[order_a.clone(), order_b.clone()], "OL-FAIL");
12755 client.submit_order_list(command).unwrap();
12756
12757 for expected in [order_a.client_order_id(), order_b.client_order_id()] {
12758 match recv_order_event(&mut rx).await {
12759 OrderEventAny::Submitted(e) => assert_eq!(e.client_order_id, expected),
12760 other => panic!("expected Submitted, was {other:?}"),
12761 }
12762 }
12763
12764 for expected in [order_b.client_order_id(), order_a.client_order_id()] {
12765 match recv_order_event(&mut rx).await {
12766 OrderEventAny::Rejected(e) => {
12767 assert_eq!(e.client_order_id, expected);
12768 assert!(e.reason.contains("Lighter submit_order dispatch failed"));
12769 }
12770 other => panic!("expected Rejected, was {other:?}"),
12771 }
12772 }
12773
12774 assert!(client.dispatch.cloid_map.get(&index_a).is_none());
12775 assert!(client.dispatch.cloid_map.get(&index_b).is_none());
12776 assert_nonce_reusable(&client.dispatch);
12777 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
12778 }
12779
12780 #[rstest]
12781 #[case::spill(16)]
12782 #[case::three(31)]
12783 #[tokio::test]
12784 async fn submit_order_list_chunks_large_lists_on_send_failure(#[case] count: usize) {
12785 let (client, cache, mut rx) = create_execution_client();
12786 let instrument_id = register_test_instrument(&client, &cache);
12787 let mut factory = test_order_factory();
12788 let mut orders = Vec::new();
12789
12790 for i in 0..count {
12791 let order = test_limit_order(&mut factory, instrument_id, &format!("O-LIST-MAX-{i}"));
12792 cache_order(&cache, order.clone());
12793 orders.push(order);
12794 }
12795
12796 let command = submit_order_list_command(&orders, "OL-MAX");
12797 client.submit_order_list(command).unwrap();
12798
12799 for chunk in orders.chunks(LIGHTER_MAX_BATCH_TX) {
12800 for order in chunk {
12801 match recv_order_event(&mut rx).await {
12802 OrderEventAny::Submitted(e) => {
12803 assert_eq!(e.client_order_id, order.client_order_id());
12804 }
12805 other => panic!("expected Submitted, was {other:?}"),
12806 }
12807 }
12808
12809 for order in chunk.iter().rev() {
12810 match recv_order_event(&mut rx).await {
12811 OrderEventAny::Rejected(e) => {
12812 assert_eq!(e.client_order_id, order.client_order_id());
12813 assert!(e.reason.contains("Lighter submit_order dispatch failed"));
12814 }
12815 other => panic!("expected Rejected, was {other:?}"),
12816 }
12817 }
12818 }
12819
12820 assert!(
12821 tokio::time::timeout(Duration::from_millis(50), rx.recv())
12822 .await
12823 .is_err(),
12824 "chunked send failure must not emit extra events",
12825 );
12826
12827 for order in &orders {
12828 let index = client
12829 .dispatch
12830 .derive_client_order_index(&order.client_order_id());
12831 assert!(client.dispatch.cloid_map.get(&index).is_none());
12832 }
12833
12834 assert_nonce_reusable(&client.dispatch);
12835 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
12836 }
12837
12838 #[tokio::test]
12839 async fn submit_order_list_denies_unsupported_order_and_dispatches_supported() {
12840 let (client, cache, mut rx) = create_execution_client();
12841 let instrument_id = register_test_instrument(&client, &cache);
12842 let mut factory = test_order_factory();
12843 let valid = test_limit_order(&mut factory, instrument_id, "O-LIST-VALID");
12844 let unsupported = factory.limit(
12845 instrument_id,
12846 OrderSide::Buy,
12847 Quantity::from("0.1000"),
12848 Price::from("2361.31"),
12849 Some(TimeInForce::Gtc),
12850 None,
12851 Some(false),
12852 Some(false),
12853 None,
12854 Some(Quantity::from("0.0500")),
12855 None,
12856 None,
12857 None,
12858 None,
12859 None,
12860 Some(ClientOrderId::from("O-LIST-ICEBERG")),
12861 );
12862 let unsupported_index = client
12863 .dispatch
12864 .derive_client_order_index(&unsupported.client_order_id());
12865 cache_order(&cache, valid.clone());
12866 cache_order(&cache, unsupported.clone());
12867
12868 let command =
12869 submit_order_list_command(&[unsupported.clone(), valid.clone()], "OL-PARTIAL");
12870 client.submit_order_list(command).unwrap();
12871
12872 match recv_order_event(&mut rx).await {
12873 OrderEventAny::Denied(e) => {
12874 assert_eq!(e.client_order_id, unsupported.client_order_id());
12875 assert!(e.reason.contains("display_qty"));
12876 }
12877 other => panic!("expected Denied, was {other:?}"),
12878 }
12879
12880 match recv_order_event(&mut rx).await {
12881 OrderEventAny::Submitted(e) => assert_eq!(e.client_order_id, valid.client_order_id()),
12882 other => panic!("expected Submitted, was {other:?}"),
12883 }
12884
12885 match recv_order_event(&mut rx).await {
12886 OrderEventAny::Rejected(e) => {
12887 assert_eq!(e.client_order_id, valid.client_order_id());
12888 assert!(e.reason.contains("Lighter submit_order dispatch failed"),);
12889 }
12890 other => panic!("expected Rejected, was {other:?}"),
12891 }
12892
12893 assert!(client.dispatch.cloid_map.get(&unsupported_index).is_none());
12894 assert_nonce_reusable(&client.dispatch);
12895 }
12896
12897 #[rstest]
12898 #[case(LighterGroupingType::OneTriggersTheOther)]
12899 #[case(LighterGroupingType::OneCancelsTheOther)]
12900 #[case(LighterGroupingType::OneTriggersOneCancelsTheOther)]
12901 fn prepare_native_group_preserves_relationships_and_sizes_children(
12902 #[case] grouping: LighterGroupingType,
12903 ) {
12904 let (client, cache, _rx) = create_execution_client();
12905 let instrument = register_test_instrument(&client, &cache);
12906 let mut orders = test_native_group(instrument, grouping);
12907 orders.reverse();
12908 let plan = client.prepare_create_group_plan(&orders, 50, None).unwrap();
12909 let context = client
12910 .fanout_dispatch_context(client.credential.as_ref().unwrap())
12911 .unwrap();
12912 let signed = context.sign_create_group(&plan).unwrap();
12913 let json: serde_json::Value = serde_json::from_str(signed.tx_info.get()).unwrap();
12914 let bodies = json["Orders"].as_array().unwrap();
12915 let has_parent = grouping != LighterGroupingType::OneCancelsTheOther;
12916 assert_eq!(plan.grouping_type, grouping);
12917 assert_eq!(signed.tx_type, LighterTxType::CreateGroupedOrders as u8);
12918 assert_eq!(bodies.len(), orders.len());
12919 assert_eq!(bodies[0]["BaseAmount"], 1_000);
12920
12921 for (index, (body, leg)) in bodies.iter().zip(&plan.orders).enumerate() {
12922 assert_eq!(body["MarketIndex"], TEST_MARKET_INDEX);
12923 assert_eq!(
12924 body["ClientOrderIndex"],
12925 client
12926 .dispatch
12927 .client_order_index(&leg.order.client_order_id())
12928 .unwrap()
12929 );
12930 assert_eq!(body["Price"], leg.price);
12931 assert_eq!(body["Type"], leg.order_type);
12932 assert_eq!(body["IsAsk"], u8::from(index > 0 || !has_parent));
12933 assert_eq!(body["ReduceOnly"], u8::from(index > 0 || !has_parent));
12934 assert_eq!(body["TimeInForce"], leg.time_in_force);
12935 assert_eq!(body["TriggerPrice"], leg.trigger_price);
12936 assert_eq!(body["OrderExpiry"], leg.order_expiry);
12937 assert_eq!(
12938 body["BaseAmount"],
12939 if has_parent && index > 0 { 0 } else { 1_000 }
12940 );
12941 assert_eq!(leg.order.quantity(), Quantity::from("0.1000"));
12942 }
12943
12944 assert_eq!(signed.pending.kind.creates().count(), orders.len());
12945 assert_eq!(
12946 client
12947 .dispatch
12948 .nonce_manager
12949 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
12950 Some(TEST_NEXT_NONCE)
12951 );
12952 }
12953
12954 #[tokio::test]
12955 async fn submit_native_group_send_failure_rejects_every_leg_and_releases_one_nonce() {
12956 let (client, cache, mut rx) = create_execution_client();
12957 let instrument = register_test_instrument(&client, &cache);
12958 let orders = test_native_group(
12959 instrument,
12960 LighterGroupingType::OneTriggersOneCancelsTheOther,
12961 );
12962
12963 for order in &orders {
12964 cache_order(&cache, order.clone());
12965 }
12966
12967 client
12968 .submit_order_list(submit_order_list_command(&orders, "OL-GROUP"))
12969 .unwrap();
12970
12971 for order in &orders {
12972 match recv_order_event(&mut rx).await {
12973 OrderEventAny::Submitted(event) => {
12974 assert_eq!(event.client_order_id, order.client_order_id());
12975 }
12976 event => panic!("expected submitted event, was {event:?}"),
12977 }
12978 }
12979
12980 for order in &orders {
12981 match recv_order_event(&mut rx).await {
12982 OrderEventAny::Rejected(event) => {
12983 assert_eq!(event.client_order_id, order.client_order_id());
12984 assert!(
12985 event
12986 .reason
12987 .contains("Lighter submit_order dispatch failed")
12988 );
12989 }
12990 event => panic!("expected rejected event, was {event:?}"),
12991 }
12992
12993 assert!(
12994 client
12995 .dispatch
12996 .order_identity(&order.client_order_id())
12997 .is_none()
12998 );
12999 }
13000
13001 assert_eq!(client.dispatch.cloid_map.len(), 0);
13002 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
13003 assert_nonce_reusable(&client.dispatch);
13004 }
13005
13006 #[tokio::test]
13007 async fn native_group_denies_default_bracket_without_dispatch() {
13008 let (client, cache, mut rx) = create_execution_client();
13009 let instrument = register_test_instrument(&client, &cache);
13010 let orders = test_order_factory()
13011 .bracket()
13012 .instrument_id(instrument)
13013 .order_side(OrderSide::Buy)
13014 .quantity(Quantity::from("0.1000"))
13015 .entry_order_type(OrderType::Limit)
13016 .entry_price(Price::from("2300.00"))
13017 .tp_price(Price::from("2400.00"))
13018 .sl_trigger_price(Price::from("2200.00"))
13019 .call();
13020
13021 for order in &orders {
13022 cache_order(&cache, order.clone());
13023 }
13024
13025 client
13026 .submit_order_list(submit_order_list_command(
13027 &orders,
13028 orders[0].order_list_id().unwrap().as_str(),
13029 ))
13030 .unwrap();
13031
13032 for order in &orders {
13033 match recv_order_event(&mut rx).await {
13034 OrderEventAny::Denied(event) => {
13035 assert_eq!(event.client_order_id, order.client_order_id());
13036 assert_eq!(
13037 event.reason.as_str(),
13038 "UNSUPPORTED_ORDER_LIST: Lighter grouped submit failed: Lighter native groups do not support Nautilus OUO quantity updates"
13039 );
13040 }
13041 event => panic!("expected denied event, was {event:?}"),
13042 }
13043 }
13044
13045 assert_eq!(client.dispatch.cloid_map.len(), 0);
13046 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
13047 assert_nonce_reusable(&client.dispatch);
13048 }
13049
13050 #[rstest]
13051 #[case(LighterGroupingType::OneTriggersTheOther)]
13052 #[case(LighterGroupingType::OneCancelsTheOther)]
13053 #[case(LighterGroupingType::OneTriggersOneCancelsTheOther)]
13054 #[tokio::test]
13055 async fn native_group_denies_quote_quantity_without_dispatch(
13056 #[case] grouping: LighterGroupingType,
13057 #[values(false, true)] first_leg: bool,
13058 ) {
13059 let (client, cache, mut rx) = create_execution_client();
13060 let instrument = register_test_instrument(&client, &cache);
13061 let mut orders = test_native_group(instrument, grouping);
13062
13063 let index = if first_leg { 0 } else { orders.len() - 1 };
13064 let order = &orders[index];
13065 let mut builder = OrderTestBuilder::new(order.order_type());
13066 builder
13067 .trader_id(order.trader_id())
13068 .strategy_id(order.strategy_id())
13069 .instrument_id(instrument)
13070 .client_order_id(order.client_order_id())
13071 .side(order.order_side())
13072 .quantity(order.quantity())
13073 .price(order.price().unwrap())
13074 .reduce_only(order.is_reduce_only())
13075 .quote_quantity(true)
13076 .order_list_id(order.order_list_id().unwrap());
13077
13078 if let Some(price) = order.trigger_price() {
13079 builder.trigger_price(price);
13080 }
13081
13082 if let Some(parent) = order.parent_order_id() {
13083 builder.parent_order_id(parent);
13084 }
13085
13086 if let Some(contingency) = order.contingency_type() {
13087 builder.contingency_type(contingency);
13088 }
13089
13090 if let Some(linked) = order.linked_order_ids() {
13091 builder.linked_order_ids(linked.to_vec());
13092 }
13093
13094 orders[index] = builder.build();
13095
13096 for order in &orders {
13097 cache_order(&cache, order.clone());
13098 }
13099
13100 client
13101 .submit_order_list(submit_order_list_command(&orders, "OL-GROUP"))
13102 .unwrap();
13103
13104 for order in &orders {
13105 match recv_order_event(&mut rx).await {
13106 OrderEventAny::Denied(event) => {
13107 assert_eq!(event.client_order_id, order.client_order_id());
13108 assert_eq!(event.reason.as_str(), "UNSUPPORTED_QUOTE_QUANTITY");
13109 }
13110 event => panic!("expected denied event, was {event:?}"),
13111 }
13112 }
13113
13114 assert!(matches!(
13115 rx.try_recv(),
13116 Err(tokio::sync::mpsc::error::TryRecvError::Empty)
13117 ));
13118 assert_eq!(client.dispatch.cloid_map.len(), 0);
13119 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
13120 assert_nonce_reusable(&client.dispatch);
13121 }
13122
13123 #[rstest]
13124 #[case(true)]
13125 #[case(false)]
13126 fn native_group_preserves_explicit_gtd_expiries(#[case] matching: bool) {
13127 let (client, cache, _rx) = create_execution_client();
13128 let instrument = register_test_instrument(&client, &cache);
13129 let mut orders = test_native_group(instrument, LighterGroupingType::OneCancelsTheOther);
13130 let expiry = client.clock.get_time_ns() + DurationNanos::from_mins(10);
13131
13132 for (index, order) in orders.iter_mut().enumerate() {
13133 *order = OrderTestBuilder::new(order.order_type())
13134 .trader_id(order.trader_id())
13135 .strategy_id(order.strategy_id())
13136 .instrument_id(instrument)
13137 .client_order_id(order.client_order_id())
13138 .side(order.order_side())
13139 .quantity(order.quantity())
13140 .price(order.price().unwrap())
13141 .trigger_price(order.trigger_price().unwrap())
13142 .reduce_only(true)
13143 .order_list_id(order.order_list_id().unwrap())
13144 .contingency_type(ContingencyType::Oco)
13145 .linked_order_ids(order.linked_order_ids().unwrap().to_vec())
13146 .time_in_force(TimeInForce::Gtd)
13147 .expire_time(UnixNanos::from(
13148 expiry.as_u64() + u64::from(!matching) * index as u64 * 1_000_000_000,
13149 ))
13150 .build();
13151 }
13152
13153 let result = client.prepare_create_group_plan(&orders, 50, None);
13154 if matching {
13155 let plan = result.unwrap();
13156 assert_eq!(
13157 plan.orders
13158 .iter()
13159 .map(|leg| leg.order_expiry)
13160 .collect::<Vec<_>>(),
13161 vec![expiry.as_u64() as i64 / 1_000_000; 2]
13162 );
13163 } else {
13164 assert_eq!(
13165 result.err().unwrap().to_string(),
13166 "Lighter group children require matching nonzero expiries"
13167 );
13168 }
13169
13170 assert_eq!(client.dispatch.cloid_map.len(), 0);
13171 }
13172
13173 #[rstest]
13174 fn native_group_denies_quantity_mismatch_without_registering_orders() {
13175 let (client, cache, _rx) = create_execution_client();
13176 let instrument = register_test_instrument(&client, &cache);
13177 let mut orders = test_native_group(
13178 instrument,
13179 LighterGroupingType::OneTriggersOneCancelsTheOther,
13180 );
13181 orders[1].set_quantity(Quantity::from("0.2000"));
13182 let error = client
13183 .prepare_create_group_plan(&orders, 50, None)
13184 .err()
13185 .unwrap();
13186 assert_eq!(
13187 error.to_string(),
13188 "Lighter group quantities must match before parent-fill sizing"
13189 );
13190 assert_eq!(client.dispatch.cloid_map.len(), 0);
13191 }
13192
13193 #[rstest]
13194 fn native_group_denies_duplicate_client_ids_without_registering_orders() {
13195 let (client, cache, _rx) = create_execution_client();
13196 let instrument = register_test_instrument(&client, &cache);
13197 let mut orders = test_native_group(
13198 instrument,
13199 LighterGroupingType::OneTriggersOneCancelsTheOther,
13200 );
13201 orders[1] = orders[2].clone();
13202 let error = client
13203 .prepare_create_group_plan(&orders, 50, None)
13204 .err()
13205 .unwrap();
13206 assert_eq!(
13207 error.to_string(),
13208 "Lighter groups require unique client order IDs"
13209 );
13210 assert_eq!(client.dispatch.cloid_map.len(), 0);
13211 }
13212
13213 #[rstest]
13214 fn native_group_reports_keep_client_links_and_held_quantities() {
13215 let (client, cache, _rx) = create_execution_client();
13216 let instrument = register_test_instrument(&client, &cache);
13217 let orders = test_native_group(
13218 instrument,
13219 LighterGroupingType::OneTriggersOneCancelsTheOther,
13220 );
13221 let plan = client.prepare_create_group_plan(&orders, 50, None).unwrap();
13222 let context = client
13223 .fanout_dispatch_context(client.credential.as_ref().unwrap())
13224 .unwrap();
13225 let signed = context.sign_create_group(&plan).unwrap();
13226 context.enqueue_batch_tx(&signed);
13227 let venues = [
13228 VenueOrderId::from("281477001"),
13229 VenueOrderId::from("281477002"),
13230 VenueOrderId::from("281477003"),
13231 ];
13232 let indices: Vec<_> = signed
13233 .pending
13234 .kind
13235 .creates()
13236 .map(|(_, index)| index)
13237 .collect();
13238
13239 for (index, order) in orders.iter().enumerate() {
13240 assert!(client.dispatch.observe_create_submission(
13241 &order.client_order_id(),
13242 indices[index],
13243 signed.pending.nonce,
13244 venues[index]
13245 ));
13246 }
13247
13248 let mut parent = reconciliation_raw_order(indices[0], venues[0], LighterOrderStatus::Open);
13249 parent.to_trigger_order_id_0 = venues[1].to_string();
13250 parent.to_trigger_order_id_1 = venues[2].to_string();
13251 let parent_report = parse_http_order_to_report(
13252 &parent,
13253 &client.registry,
13254 account_id(),
13255 UnixNanos::from(7),
13256 &client.dispatch,
13257 )
13258 .unwrap();
13259 let parent_report = client.dispatch.translate_order_cloid(parent_report);
13260 let mut child =
13261 reconciliation_raw_order(indices[1], venues[1], LighterOrderStatus::Pending);
13262 child.order_type = LighterOrderKind::StopLossLimit;
13263 child.parent_order_id = venues[0].to_string();
13264 child.to_cancel_order_id_0 = venues[2].to_string();
13265 child.initial_base_amount = Decimal::ZERO;
13266 child.filled_base_amount = Decimal::ZERO;
13267 child.trigger_price = Decimal::from(2200);
13268 let child_report = parse_http_order_to_report(
13269 &child,
13270 &client.registry,
13271 account_id(),
13272 UnixNanos::from(8),
13273 &client.dispatch,
13274 )
13275 .unwrap();
13276 let child_report = client.dispatch.translate_order_cloid(child_report);
13277 client
13278 .dispatch
13279 .retire_order_identity(&orders[0].client_order_id());
13280 child.parent_order_id = "0".to_string();
13281 child.to_cancel_order_id_0 = "0".to_string();
13282 child.status = LighterOrderStatus::Open;
13283 child.trigger_status = LighterTriggerStatus::Na;
13284 child.trigger_price = Decimal::ZERO;
13285 child.initial_base_amount = Decimal::from_str("0.0400").unwrap();
13286 child.remaining_base_amount = child.initial_base_amount;
13287 let activated = parse_http_order_to_report(
13288 &child,
13289 &client.registry,
13290 account_id(),
13291 UnixNanos::from(9),
13292 &client.dispatch,
13293 )
13294 .unwrap();
13295 let activated = client.dispatch.translate_order_cloid(activated);
13296 assert_eq!(
13297 parent_report.client_order_id,
13298 Some(orders[0].client_order_id())
13299 );
13300 assert_eq!(parent_report.contingency_type, Some(ContingencyType::Oto));
13301 assert_eq!(
13302 parent_report.linked_order_ids,
13303 Some(vec![
13304 orders[1].client_order_id(),
13305 orders[2].client_order_id()
13306 ])
13307 );
13308 assert_eq!(parent_report.parent_order_id, None);
13309 assert_eq!(
13310 child_report.client_order_id,
13311 Some(orders[1].client_order_id())
13312 );
13313 assert_eq!(child_report.contingency_type, Some(ContingencyType::Oco));
13314 assert_eq!(
13315 child_report.linked_order_ids,
13316 Some(vec![orders[2].client_order_id()])
13317 );
13318 assert_eq!(
13319 child_report.parent_order_id,
13320 Some(orders[0].client_order_id())
13321 );
13322 assert_eq!(child_report.quantity, Quantity::from("0.1000"));
13323 assert_eq!(activated.order_type, OrderType::StopLimit);
13324 assert_eq!(activated.order_status, OrderStatus::Triggered);
13325 assert_eq!(activated.trigger_price, Some(Price::from("2200.00")));
13326 assert_eq!(activated.quantity, Quantity::from("0.0400"));
13327 assert_eq!(activated.parent_order_id, Some(orders[0].client_order_id()));
13328 assert_eq!(
13329 activated.linked_order_ids,
13330 Some(vec![orders[2].client_order_id()])
13331 );
13332 }
13333
13334 #[rstest]
13335 fn native_group_restores_held_child_before_first_stream_frame() {
13336 let (client, cache, mut rx) = create_execution_client();
13337 let instrument = register_test_instrument(&client, &cache);
13338 let orders = test_native_group(
13339 instrument,
13340 LighterGroupingType::OneTriggersOneCancelsTheOther,
13341 );
13342 let child = orders[1].clone();
13343 let venue = VenueOrderId::from("281477004");
13344 cache_accepted_order(&cache, child.clone(), venue, Some(client_id()));
13345 let cached = cache
13346 .borrow()
13347 .order_owned(&child.client_order_id())
13348 .unwrap();
13349 client.dispatch.seed_order_restorations(vec![cached]);
13350 let (_, index) = forced_probed_index(child.client_order_id());
13351 let mut raw = reconciliation_raw_order(index, venue, LighterOrderStatus::Pending);
13352 raw.order_type = LighterOrderKind::StopLossLimit;
13353 raw.initial_base_amount = Decimal::ZERO;
13354 raw.filled_base_amount = Decimal::ZERO;
13355 raw.is_ask = true;
13356 raw.side = Some(LighterOrderSide::Sell);
13357 raw.reduce_only = true;
13358 raw.price = Decimal::from(2190);
13359 raw.trigger_price = Decimal::from(2200);
13360 dispatch_lighter_order(
13361 &raw,
13362 &client.dispatch,
13363 &client.emitter,
13364 &client.registry,
13365 account_id(),
13366 trader_id(),
13367 UnixNanos::from(1),
13368 );
13369 let events = drain_events(&mut rx);
13370 let report = parse_http_order_to_report(
13371 &raw,
13372 &client.registry,
13373 account_id(),
13374 UnixNanos::from(2),
13375 &client.dispatch,
13376 )
13377 .unwrap();
13378 let report = client.dispatch.translate_order_cloid(report);
13379 assert!(
13380 events.is_empty(),
13381 "restored pending frame must be a tracked replay: {events:?}"
13382 );
13383 assert_eq!(
13384 cache
13385 .borrow()
13386 .order(&child.client_order_id())
13387 .unwrap()
13388 .quantity(),
13389 Quantity::from("0.1000")
13390 );
13391 assert_eq!(
13392 client.dispatch.client_order_index(&child.client_order_id()),
13393 Some(index)
13394 );
13395 assert_eq!(report.client_order_id, Some(child.client_order_id()));
13396 assert_eq!(report.quantity, Quantity::from("0.1000"));
13397 assert_eq!(report.parent_order_id, Some(orders[0].client_order_id()));
13398 assert_eq!(
13399 report.linked_order_ids,
13400 Some(vec![orders[2].client_order_id()])
13401 );
13402 assert_eq!(report.contingency_type, Some(ContingencyType::Oco));
13403 assert_eq!(report.order_type, OrderType::StopLimit);
13404 assert_eq!(report.trigger_price, Some(Price::from("2200.00")));
13405 assert_eq!(report.price, Some(Price::from("2190.00")));
13406 assert_eq!(report.order_side, Some(OrderSide::Sell));
13407 assert_eq!(report.order_status, OrderStatus::Accepted);
13408 assert!(report.reduce_only);
13409 }
13410
13411 #[rstest]
13412 #[case(true)]
13413 #[case(false)]
13414 fn native_oco_cancel_targets_peer_with_venue_or_shared_submission_index(
13415 #[case] acknowledged: bool,
13416 ) {
13417 let (client, cache, _rx) = create_execution_client();
13418 let instrument = register_test_instrument(&client, &cache);
13419 let orders = test_native_group(
13420 instrument,
13421 LighterGroupingType::OneTriggersOneCancelsTheOther,
13422 );
13423 let group = client.prepare_create_group_plan(&orders, 50, None).unwrap();
13424 let context = client
13425 .fanout_dispatch_context(client.credential.as_ref().unwrap())
13426 .unwrap();
13427 let signed = context.sign_create_group(&group).unwrap();
13428 client
13429 .dispatch
13430 .enqueue_pending_sendtx(signed.pending.clone());
13431 let first = VenueOrderId::from("281477101");
13432 let second = VenueOrderId::from("281477102");
13433 cache_accepted_order(&cache, orders[1].clone(), first, Some(client_id()));
13434 let first_index = client
13435 .dispatch
13436 .client_order_index(&orders[1].client_order_id())
13437 .unwrap();
13438 let second_index = client
13439 .dispatch
13440 .client_order_index(&orders[2].client_order_id())
13441 .unwrap();
13442 assert!(client.dispatch.observe_create_submission(
13443 &orders[1].client_order_id(),
13444 first_index,
13445 signed.pending.nonce,
13446 first
13447 ));
13448
13449 if acknowledged {
13450 cache_accepted_order(&cache, orders[2].clone(), second, Some(client_id()));
13451 assert!(client.dispatch.observe_create_submission(
13452 &orders[2].client_order_id(),
13453 second_index,
13454 signed.pending.nonce,
13455 second
13456 ));
13457 } else {
13458 cache_order(&cache, orders[2].clone());
13459 cache
13460 .borrow_mut()
13461 .update_order(&TestOrderEventStubs::submitted(&orders[2], account_id()))
13462 .unwrap();
13463 }
13464
13465 let command = CancelOrder::new(
13466 trader_id(),
13467 Some(client_id()),
13468 strategy_id(),
13469 instrument,
13470 orders[1].client_order_id(),
13471 None,
13472 UUID4::new(),
13473 UnixNanos::default(),
13474 None,
13475 None,
13476 );
13477 let plans = client.prepare_oco_cancel_plans(&command).unwrap();
13478 assert_eq!(plans.len(), 2);
13479
13480 for (index, plan) in plans.iter().enumerate() {
13481 assert_eq!(plan.client_order_id, orders[index + 1].client_order_id());
13482 assert_eq!(plan.strategy_id, strategy_id());
13483 assert_eq!(plan.instrument_id, instrument);
13484 assert_eq!(plan.market_index, TEST_MARKET_INDEX);
13485 }
13486
13487 assert_eq!(plans[0].venue_order_id, Some(first));
13488 assert_eq!(plans[0].venue_index, 281_477_101);
13489 assert_eq!(plans[1].venue_order_id, acknowledged.then_some(second));
13490 assert_eq!(
13491 plans[1].venue_index,
13492 if acknowledged {
13493 281_477_102
13494 } else {
13495 second_index
13496 }
13497 );
13498 let mut override_venue = command.clone();
13499 override_venue.venue_order_id = Some(VenueOrderId::from("281477103"));
13500 assert!(client.prepare_oco_cancel_plans(&override_venue).is_none());
13501
13502 if !acknowledged {
13503 client
13504 .dispatch
13505 .mark_order_submission(&orders[2].client_order_id(), signed.pending.nonce + 1);
13506 assert!(client.prepare_oco_cancel_plans(&command).is_none());
13507 }
13508 }
13509
13510 #[rstest]
13511 fn native_group_denies_held_child_modification() {
13512 let (client, cache, _rx) = create_execution_client();
13513 let instrument = register_test_instrument(&client, &cache);
13514 let orders = test_native_group(instrument, LighterGroupingType::OneTriggersTheOther);
13515 let child = orders[1].clone();
13516 let venue = VenueOrderId::from("281477004");
13517 cache_accepted_order(&cache, child.clone(), venue, Some(client_id()));
13518 client.dispatch.store_snapshot(
13519 child.client_order_id(),
13520 OrderShapeSnapshot {
13521 quantity: Quantity::from("0.0000"),
13522 price: child.price(),
13523 trigger_price: child.trigger_price(),
13524 },
13525 );
13526
13527 let command = ModifyOrder::new(
13528 trader_id(),
13529 Some(client_id()),
13530 strategy_id(),
13531 instrument,
13532 child.client_order_id(),
13533 Some(venue),
13534 None,
13535 Some(Price::from("2180.00")),
13536 None,
13537 UUID4::new(),
13538 UnixNanos::default(),
13539 None,
13540 None,
13541 );
13542 let error = client.prepare_modify_order_plan(&command).err().unwrap();
13543 assert_eq!(
13544 error.to_string(),
13545 "Lighter cannot modify an OTO child while it waits for parent execution"
13546 );
13547 }
13548
13549 #[rstest]
13550 fn native_group_modifies_triggered_child_without_rearming_trigger() {
13551 let (client, cache, _rx) = create_execution_client();
13552 let instrument = register_test_instrument(&client, &cache);
13553 let orders = test_native_group(
13554 instrument,
13555 LighterGroupingType::OneTriggersOneCancelsTheOther,
13556 );
13557 let child = orders[2].clone();
13558 let venue = VenueOrderId::from("281477004");
13559 cache_accepted_order(&cache, child.clone(), venue, Some(client_id()));
13560 client
13561 .dispatch
13562 .mark_triggered_emitted(child.client_order_id());
13563
13564 let command = ModifyOrder::new(
13565 trader_id(),
13566 Some(client_id()),
13567 strategy_id(),
13568 instrument,
13569 child.client_order_id(),
13570 Some(venue),
13571 None,
13572 Some(Price::from("2380.00")),
13573 None,
13574 UUID4::new(),
13575 UnixNanos::default(),
13576 None,
13577 None,
13578 );
13579 let plan = client.prepare_modify_order_plan(&command).unwrap();
13580 let mut rearm = command;
13581 rearm.trigger_price = Some(Price::from("2450.00"));
13582 let error = client.prepare_modify_order_plan(&rearm).err().unwrap();
13583 assert_eq!(plan.trigger_price, 0);
13584 assert_eq!(plan.price, 238_000);
13585 assert_eq!(plan.base_amount, 1_000);
13586 assert_eq!(
13587 error.to_string(),
13588 "Lighter cannot modify a trigger price after the order has triggered"
13589 );
13590 }
13591
13592 fn test_native_group(
13593 instrument_id: InstrumentId,
13594 grouping: LighterGroupingType,
13595 ) -> Vec<OrderAny> {
13596 let has_parent = grouping != LighterGroupingType::OneCancelsTheOther;
13597 let ids = [
13598 ClientOrderId::from("O-GROUP-ENTRY"),
13599 ClientOrderId::from("O-GROUP-SL"),
13600 ClientOrderId::from("O-GROUP-TP"),
13601 ];
13602 let mut orders = Vec::new();
13603 if has_parent {
13604 orders.push(
13605 OrderTestBuilder::new(OrderType::Limit)
13606 .trader_id(trader_id())
13607 .strategy_id(strategy_id())
13608 .instrument_id(instrument_id)
13609 .client_order_id(ids[0])
13610 .side(OrderSide::Buy)
13611 .quantity(Quantity::from("0.1000"))
13612 .price(Price::from("2300.00"))
13613 .order_list_id(OrderListId::from("OL-GROUP"))
13614 .contingency_type(ContingencyType::Oto)
13615 .linked_order_ids(if grouping == LighterGroupingType::OneTriggersTheOther {
13616 vec![ids[1]]
13617 } else {
13618 vec![ids[1], ids[2]]
13619 })
13620 .build(),
13621 );
13622 }
13623
13624 let count = if grouping == LighterGroupingType::OneTriggersTheOther {
13625 1
13626 } else {
13627 2
13628 };
13629
13630 let children = [
13631 (
13632 OrderType::StopLimit,
13633 Price::from("2190.00"),
13634 Price::from("2200.00"),
13635 ),
13636 (
13637 OrderType::LimitIfTouched,
13638 Price::from("2390.00"),
13639 Price::from("2400.00"),
13640 ),
13641 ];
13642
13643 for (index, (kind, price, trigger)) in children.into_iter().take(count).enumerate() {
13644 let mut builder = OrderTestBuilder::new(kind);
13645 builder
13646 .trader_id(trader_id())
13647 .strategy_id(strategy_id())
13648 .instrument_id(instrument_id)
13649 .client_order_id(ids[index + 1])
13650 .side(OrderSide::Sell)
13651 .quantity(Quantity::from("0.1000"))
13652 .price(price)
13653 .trigger_price(trigger)
13654 .reduce_only(true)
13655 .order_list_id(OrderListId::from("OL-GROUP"));
13656
13657 if has_parent {
13658 builder.parent_order_id(ids[0]);
13659 }
13660
13661 if count == 2 {
13662 builder
13663 .contingency_type(ContingencyType::Oco)
13664 .linked_order_ids(vec![ids[2 - index]]);
13665 }
13666
13667 orders.push(builder.build());
13668 }
13669
13670 orders
13671 }
13672
13673 #[tokio::test]
13674 async fn submit_order_list_plain_limit_oco_denies_all_without_dispatch() {
13675 let (client, cache, mut rx) = create_execution_client();
13676 let instrument_id = register_test_instrument(&client, &cache);
13677 let order_a_id = "O-LIST-OCO-A";
13678 let order_b_id = "O-LIST-OCO-B";
13679 let order_a = test_contingent_limit_order(instrument_id, order_a_id, "OL-OCO", order_b_id);
13680 let order_b = test_contingent_limit_order(instrument_id, order_b_id, "OL-OCO", order_a_id);
13681 cache_order(&cache, order_a.clone());
13682 cache_order(&cache, order_b.clone());
13683
13684 let command = submit_order_list_command(&[order_a.clone(), order_b.clone()], "OL-OCO");
13685 client.submit_order_list(command).unwrap();
13686
13687 for order in [&order_a, &order_b] {
13688 match recv_order_event(&mut rx).await {
13689 OrderEventAny::Denied(e) => {
13690 assert_eq!(e.client_order_id, order.client_order_id());
13691 assert!(e.reason.contains("reduce-only SL/TP orders"));
13692 }
13693 other => panic!("expected Denied, was {other:?}"),
13694 }
13695 }
13696
13697 assert!(
13698 tokio::time::timeout(Duration::from_millis(50), rx.recv())
13699 .await
13700 .is_err(),
13701 "grouped list denial must not emit extra events",
13702 );
13703 assert_nonce_reusable(&client.dispatch);
13704 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
13705 }
13706
13707 #[tokio::test]
13708 async fn cancel_order_send_failure_emits_cancel_rejected_and_rolls_back() {
13709 let (client, cache, mut rx) = create_execution_client();
13710 let instrument_id = register_test_instrument(&client, &cache);
13711 let mut factory = test_order_factory();
13712 let order = test_limit_order(&mut factory, instrument_id, "O-CANCEL-FAIL");
13713 let client_order_id = order.client_order_id();
13714 let venue_order_id = VenueOrderId::from("123");
13715 cache_pending_cancel_order(&cache, order, venue_order_id);
13716
13717 let command = CancelOrder::new(
13718 trader_id(),
13719 Some(client_id()),
13720 strategy_id(),
13721 instrument_id,
13722 client_order_id,
13723 Some(venue_order_id),
13724 UUID4::new(),
13725 UnixNanos::default(),
13726 None,
13727 None,
13728 );
13729 client.cancel_order(command).unwrap();
13730
13731 let rejected = recv_order_event(&mut rx).await;
13732
13733 match rejected {
13734 OrderEventAny::CancelRejected(event) => {
13735 assert_eq!(event.client_order_id, client_order_id);
13736 assert_eq!(event.instrument_id, instrument_id);
13737 assert_eq!(event.venue_order_id, Some(venue_order_id));
13738 assert!(
13739 event
13740 .reason
13741 .contains("Lighter cancel_order dispatch failed"),
13742 );
13743 assert!(event.reason.contains("handler unavailable"));
13744 }
13745 event => panic!("expected cancel rejected event, was {event:?}"),
13746 }
13747
13748 assert_nonce_reusable(&client.dispatch);
13749 assert_eq!(
13750 client.dispatch.pending_sendtx_len(),
13751 0,
13752 "local-send-failure must remove the pending cancel entry",
13753 );
13754 assert_eq!(client.dispatch.pending_order_action(&client_order_id), None);
13755 }
13756
13757 #[tokio::test]
13758 async fn cancel_order_unknown_send_outcome_retains_pending_state() {
13759 let (client, cache, mut rx) = create_execution_client();
13760 let instrument_id = register_test_instrument(&client, &cache);
13761 let mut factory = test_order_factory();
13762 let order = test_limit_order(&mut factory, instrument_id, "O-CANCEL-UNKNOWN");
13763 let client_order_id = order.client_order_id();
13764 let venue_order_id = VenueOrderId::from("123");
13765 cache_pending_cancel_order(&cache, order, venue_order_id);
13766 client.ws_client.drop_next_send_tx_result_for_test().await;
13767
13768 client
13769 .cancel_order(CancelOrder::new(
13770 trader_id(),
13771 Some(client_id()),
13772 strategy_id(),
13773 instrument_id,
13774 client_order_id,
13775 Some(venue_order_id),
13776 UUID4::new(),
13777 UnixNanos::default(),
13778 None,
13779 None,
13780 ))
13781 .unwrap();
13782
13783 assert!(
13784 tokio::time::timeout(Duration::from_millis(50), rx.recv())
13785 .await
13786 .is_err(),
13787 "unknown outcome must not emit OrderCancelRejected",
13788 );
13789 assert_eq!(client.dispatch.pending_sendtx_len(), 1);
13790 assert_eq!(
13791 client.dispatch.pending_order_action(&client_order_id),
13792 Some(PendingOrderAction::Cancel),
13793 );
13794 assert_eq!(
13795 client
13796 .dispatch
13797 .nonce_manager
13798 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
13799 Some(TEST_NEXT_NONCE),
13800 );
13801 }
13802
13803 #[tokio::test]
13804 async fn cancel_order_prepare_failure_emits_cancel_rejected_without_dispatch() {
13805 let (client, cache, mut rx) = create_execution_client();
13806 let instrument_id = register_test_instrument(&client, &cache);
13807 let mut factory = test_order_factory();
13808 let order = test_limit_order(&mut factory, instrument_id, "O-CANCEL-NO-VOI");
13809 let client_order_id = order.client_order_id();
13810 cache_pending_cancel_order(&cache, order, VenueOrderId::from("123"));
13811
13812 let command = CancelOrder::new(
13813 trader_id(),
13814 Some(client_id()),
13815 strategy_id(),
13816 instrument_id,
13817 client_order_id,
13818 None,
13819 UUID4::new(),
13820 UnixNanos::default(),
13821 None,
13822 None,
13823 );
13824 client.cancel_order(command).unwrap();
13825
13826 let rejected = recv_order_event(&mut rx).await;
13827 match rejected {
13828 OrderEventAny::CancelRejected(event) => {
13829 assert_eq!(event.client_order_id, client_order_id);
13830 assert_eq!(event.instrument_id, instrument_id);
13831 assert_eq!(event.venue_order_id, None);
13832 assert!(event.reason.contains("Lighter cancel_order failed"));
13833 assert!(event.reason.contains("venue order_id not yet known"));
13834 }
13835 event => panic!("expected cancel rejected event, was {event:?}"),
13836 }
13837
13838 assert_nonce_reusable(&client.dispatch);
13839 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
13840 assert_eq!(client.dispatch.pending_order_action(&client_order_id), None);
13841 assert!(
13842 tokio::time::timeout(Duration::from_millis(50), rx.recv())
13843 .await
13844 .is_err(),
13845 "prepare failure must emit exactly one cancel rejection",
13846 );
13847 }
13848
13849 #[tokio::test]
13850 async fn cancel_order_prepare_failure_emits_cancel_rejected_when_order_uncached() {
13851 let (client, cache, mut rx) = create_execution_client();
13852 let instrument_id = register_test_instrument(&client, &cache);
13853 let client_order_id = ClientOrderId::from("O-CANCEL-NO-CACHE");
13854 let venue_order_id = VenueOrderId::from("123");
13855
13856 let command = CancelOrder::new(
13857 trader_id(),
13858 Some(client_id()),
13859 strategy_id(),
13860 instrument_id,
13861 client_order_id,
13862 Some(venue_order_id),
13863 UUID4::new(),
13864 UnixNanos::default(),
13865 None,
13866 None,
13867 );
13868 client.cancel_order(command).unwrap();
13869
13870 let rejected = recv_order_event(&mut rx).await;
13871 match rejected {
13872 OrderEventAny::CancelRejected(event) => {
13873 assert_eq!(event.client_order_id, client_order_id);
13874 assert_eq!(event.instrument_id, instrument_id);
13875 assert_eq!(event.venue_order_id, Some(venue_order_id));
13876 assert!(event.reason.contains("Lighter cancel_order failed"));
13877 assert!(event.reason.contains("order not found in cache"));
13878 }
13879 event => panic!("expected cancel rejected event, was {event:?}"),
13880 }
13881
13882 assert_nonce_reusable(&client.dispatch);
13883 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
13884 assert!(
13885 tokio::time::timeout(Duration::from_millis(50), rx.recv())
13886 .await
13887 .is_err(),
13888 "prepare failure must emit exactly one cancel rejection",
13889 );
13890 }
13891
13892 #[rstest]
13893 #[case::buy(Some(OrderSide::Buy), false, vec![0, 2])]
13894 #[case::sell(Some(OrderSide::Sell), false, vec![1, 3])]
13895 #[case::unsided(None, false, vec![0, 1, 2, 3])]
13896 #[case::empty_buy(Some(OrderSide::Buy), true, vec![])]
13897 #[case::empty_sell(Some(OrderSide::Sell), true, vec![])]
13898 #[case::empty_unsided(None, true, vec![])]
13899 #[tokio::test]
13900 async fn cancel_all_orders_filters_orders_and_preserves_owners(
13901 #[case] order_side: Option<OrderSide>,
13902 #[case] empty: bool,
13903 #[case] expected_indices: Vec<usize>,
13904 ) {
13905 let (client, cache, mut rx) = create_execution_client();
13906 let instrument_id = register_test_instrument(&client, &cache);
13907 let other_instrument_id = InstrumentId::from("BTC-PERP.LIGHTER");
13908 let selected_account = account_id();
13909 let other_account = AccountId::from("LIGHTER-002");
13910 let mut orders = Vec::new();
13911
13912 for (index, (instrument, owner, side, open, account)) in [
13913 (
13914 instrument_id,
13915 "S-001",
13916 OrderSide::Buy,
13917 true,
13918 selected_account,
13919 ),
13920 (
13921 instrument_id,
13922 "S-001",
13923 OrderSide::Sell,
13924 true,
13925 selected_account,
13926 ),
13927 (
13928 instrument_id,
13929 "S-002",
13930 OrderSide::Buy,
13931 true,
13932 selected_account,
13933 ),
13934 (
13935 instrument_id,
13936 "S-002",
13937 OrderSide::Sell,
13938 true,
13939 selected_account,
13940 ),
13941 (
13942 other_instrument_id,
13943 "S-001",
13944 OrderSide::Buy,
13945 true,
13946 selected_account,
13947 ),
13948 (
13949 other_instrument_id,
13950 "S-002",
13951 OrderSide::Sell,
13952 true,
13953 selected_account,
13954 ),
13955 (
13956 instrument_id,
13957 "S-001",
13958 OrderSide::Buy,
13959 false,
13960 selected_account,
13961 ),
13962 (
13963 instrument_id,
13964 "S-002",
13965 OrderSide::Sell,
13966 false,
13967 selected_account,
13968 ),
13969 (instrument_id, "S-001", OrderSide::Buy, true, other_account),
13970 (instrument_id, "S-002", OrderSide::Sell, true, other_account),
13971 ]
13972 .into_iter()
13973 .enumerate()
13974 {
13975 if empty
13976 && account == selected_account
13977 && instrument == instrument_id
13978 && open
13979 && order_side.is_none_or(|filter| filter == side)
13980 {
13981 continue;
13982 }
13983
13984 let order = OrderTestBuilder::new(OrderType::Limit)
13985 .trader_id(trader_id())
13986 .strategy_id(StrategyId::from(owner))
13987 .instrument_id(instrument)
13988 .client_order_id(ClientOrderId::from(format!("O-CANCEL-ALL-{index}")))
13989 .side(side)
13990 .quantity(Quantity::from("0.1000"))
13991 .price(Price::from("2361.31"))
13992 .build();
13993 let venue_order_id = VenueOrderId::from(format!("{}", 123 + index));
13994 let accepted = TestOrderEventStubs::accepted(&order, account, venue_order_id);
13995 cache_order(&cache, order.clone());
13996 cache.borrow_mut().update_order(&accepted).unwrap();
13997 client
13998 .dispatch
13999 .venue_id_map
14000 .insert(order.client_order_id(), venue_order_id);
14001
14002 let event = if open {
14003 OrderEventAny::PendingCancel(OrderPendingCancel::new(
14004 order.trader_id(),
14005 order.strategy_id(),
14006 instrument,
14007 order.client_order_id(),
14008 Some(account),
14009 UUID4::new(),
14010 UnixNanos::default(),
14011 UnixNanos::default(),
14012 false,
14013 Some(venue_order_id),
14014 ))
14015 } else {
14016 TestOrderEventStubs::canceled(&order, account, Some(venue_order_id))
14017 };
14018
14019 cache.borrow_mut().update_order(&event).unwrap();
14020 orders.push((order, venue_order_id));
14021 }
14022
14023 client
14024 .cancel_all_orders(CancelAllOrders::new(
14025 trader_id(),
14026 Some(client_id()),
14027 strategy_id(),
14028 instrument_id,
14029 order_side,
14030 UUID4::new(),
14031 UnixNanos::default(),
14032 None,
14033 None,
14034 ))
14035 .unwrap();
14036
14037 let mut actual = Vec::new();
14040
14041 for _ in &expected_indices {
14042 match recv_order_event(&mut rx).await {
14043 OrderEventAny::CancelRejected(event) => {
14044 assert!(
14045 event
14046 .reason
14047 .contains("Lighter cancel_order dispatch failed")
14048 );
14049 assert!(event.reason.contains("handler unavailable"));
14050 assert_eq!(event.trader_id, trader_id());
14051 assert_eq!(event.account_id, Some(account_id()));
14052 actual.push((
14053 event.client_order_id,
14054 event.strategy_id,
14055 event.instrument_id,
14056 event.venue_order_id,
14057 ));
14058 }
14059 event => panic!("expected cancel rejected event, was {event:?}"),
14060 }
14061 }
14062
14063 let mut expected: Vec<_> = expected_indices
14064 .iter()
14065 .map(|&index| {
14066 let (order, venue_order_id) = &orders[index];
14067 (
14068 order.client_order_id(),
14069 order.strategy_id(),
14070 order.instrument_id(),
14071 Some(*venue_order_id),
14072 )
14073 })
14074 .collect();
14075
14076 actual.sort();
14077 expected.sort();
14078
14079 assert_eq!(actual, expected);
14080 assert!(
14081 tokio::time::timeout(Duration::from_millis(50), rx.recv())
14082 .await
14083 .is_err()
14084 );
14085 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
14086
14087 for (order, _) in orders {
14088 assert_eq!(
14089 client
14090 .dispatch
14091 .pending_order_action(&order.client_order_id()),
14092 None
14093 );
14094 }
14095
14096 if empty {
14097 assert_nonce_reusable(&client.dispatch);
14098 }
14099 }
14100
14101 #[tokio::test]
14102 async fn cancel_all_orders_prepare_failure_suppresses_cancel_rejected_for_open_order() {
14103 let (client, cache, mut rx) = create_execution_client();
14104 let instrument_id = register_test_instrument(&client, &cache);
14105 let mut factory = test_order_factory();
14106 let order = test_limit_order(&mut factory, instrument_id, "O-CANCEL-ALL-NO-VOI");
14107 cache_accepted_order(&cache, order, VenueOrderId::from("123"), Some(client_id()));
14108
14109 let command = CancelAllOrders::new(
14110 trader_id(),
14111 Some(client_id()),
14112 strategy_id(),
14113 instrument_id,
14114 Some(OrderSide::Buy),
14115 UUID4::new(),
14116 UnixNanos::default(),
14117 None,
14118 None,
14119 );
14120 client.cancel_all_orders(command).unwrap();
14121
14122 assert_nonce_reusable(&client.dispatch);
14123 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
14124 assert!(
14125 tokio::time::timeout(Duration::from_millis(50), rx.recv())
14126 .await
14127 .is_err(),
14128 "cancel-all prepare failure must not emit an invalid cancel rejection",
14129 );
14130 }
14131
14132 #[tokio::test]
14133 async fn cancel_order_nonce_prepare_failure_emits_cancel_rejected() {
14134 let mut config = test_config();
14135 config.base_url_http = Some(spawn_next_nonce_server(100).await);
14136 let (client, cache, mut rx) = create_execution_client_with_config(config);
14137 let instrument_id = register_test_instrument(&client, &cache);
14138 let mut factory = test_order_factory();
14139 let order = test_limit_order(&mut factory, instrument_id, "O-CANCEL-NONCE-FAIL");
14140 let client_order_id = order.client_order_id();
14141 let venue_order_id = VenueOrderId::from("123");
14142 cache_pending_cancel_order(&cache, order, venue_order_id);
14143
14144 let window = i64::from(client.dispatch.nonce_manager.skip_window());
14145 for _ in 0..window {
14146 client
14147 .dispatch
14148 .nonce_manager
14149 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
14150 .unwrap();
14151 }
14152
14153 let command = CancelOrder::new(
14154 trader_id(),
14155 Some(client_id()),
14156 strategy_id(),
14157 instrument_id,
14158 client_order_id,
14159 Some(venue_order_id),
14160 UUID4::new(),
14161 UnixNanos::default(),
14162 None,
14163 None,
14164 );
14165 client.cancel_order(command).unwrap();
14166
14167 let rejected = recv_order_event(&mut rx).await;
14168 match rejected {
14169 OrderEventAny::CancelRejected(event) => {
14170 assert_eq!(event.client_order_id, client_order_id);
14171 assert_eq!(event.venue_order_id, Some(venue_order_id));
14172 assert!(event.reason.contains("failed to allocate Lighter nonce"),);
14173 assert!(event.reason.contains("skip-window exhausted"));
14174 }
14175 event => panic!("expected cancel rejected event, was {event:?}"),
14176 }
14177
14178 wait_for_spawned_tasks(&client).await;
14179 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
14180 assert_eq!(
14181 client
14182 .dispatch
14183 .nonce_manager
14184 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
14185 .unwrap(),
14186 100,
14187 );
14188 }
14189
14190 #[tokio::test]
14191 async fn batch_cancel_releases_split_reservations_before_waiting_for_ack() {
14192 let (client, cache, _rx) = create_execution_client();
14193 let instrument_id = register_test_instrument(&client, &cache);
14194 client.ws_client.drop_next_send_tx_result_for_test().await;
14195 let context = client
14196 .fanout_dispatch_context(client.credential.as_ref().unwrap())
14197 .unwrap();
14198
14199 let plan = CancelOrderPlan {
14200 client_order_id: ClientOrderId::from("O-SPLIT-CANCEL"),
14201 strategy_id: strategy_id(),
14202 instrument_id,
14203 venue_order_id: Some(VenueOrderId::from("123")),
14204 market_index: TEST_MARKET_INDEX,
14205 venue_index: 123,
14206 };
14207
14208 let first = context.sign_cancel_order(&plan).unwrap();
14209 let interleaved = context.build_tx_context().unwrap();
14210 let second = context.sign_cancel_order(&plan).unwrap();
14211 let first_nonce = first.nonce;
14212 let second_nonce = second.nonce;
14213 drop(interleaved);
14214
14215 let task = tokio::spawn(async move {
14216 context
14217 .send_batch(vec![
14218 PreparedBatchTx::from((first, false)),
14219 PreparedBatchTx::from((second, false)),
14220 ])
14221 .await;
14222 });
14223
14224 wait_until_async(
14225 || async { client.dispatch.pending_sendtx_len() != 0 },
14226 Duration::from_secs(2),
14227 )
14228 .await;
14229
14230 let released = tokio::time::timeout(
14231 Duration::from_millis(250),
14232 client.nonce_submission_gate.write(),
14233 )
14234 .await
14235 .is_ok();
14236 let pending = client.dispatch.pending_sendtx.lock().clone();
14237 task.abort();
14238 let _ = task.await;
14239
14240 assert!(
14241 released,
14242 "unacknowledged group must not hold later nonce reservations"
14243 );
14244 assert_eq!(second_nonce, first_nonce + 2);
14245 assert_eq!(pending.len(), 1);
14246 assert_eq!(pending[0].nonce, first_nonce);
14247 }
14248
14249 #[rstest]
14250 #[case::not_sent(false)]
14251 #[case::ambiguous(true)]
14252 #[tokio::test]
14253 async fn batch_cancel_orders_send_failure_preserves_delivery_semantics(
14254 #[case] ambiguous: bool,
14255 ) {
14256 let (client, cache, mut rx) = create_execution_client();
14257 let instrument_id = register_test_instrument(&client, &cache);
14258 let mut factory = test_order_factory();
14259 let cancels = ["O-BATCH-CANCEL-A", "O-BATCH-CANCEL-B"]
14260 .iter()
14261 .enumerate()
14262 .map(|(i, id)| {
14263 let order = test_limit_order(&mut factory, instrument_id, id);
14264 let client_order_id = order.client_order_id();
14265 let venue_order_id = VenueOrderId::from(format!("{}", 123 + i).as_str());
14266 cache_pending_cancel_order(&cache, order, venue_order_id);
14267
14268 CancelOrder::new(
14269 trader_id(),
14270 Some(client_id()),
14271 strategy_id(),
14272 instrument_id,
14273 client_order_id,
14274 Some(venue_order_id),
14275 UUID4::new(),
14276 UnixNanos::default(),
14277 None,
14278 None,
14279 )
14280 })
14281 .collect::<Vec<_>>();
14282
14283 let command = BatchCancelOrders::new(
14284 trader_id(),
14285 Some(client_id()),
14286 strategy_id(),
14287 instrument_id,
14288 cancels.clone(),
14289 UUID4::new(),
14290 UnixNanos::default(),
14291 None,
14292 None,
14293 );
14294
14295 if ambiguous {
14296 client.ws_client.drop_next_send_tx_result_for_test().await;
14297 }
14298
14299 client.batch_cancel_orders(command).unwrap();
14300
14301 if ambiguous {
14302 assert!(
14303 tokio::time::timeout(Duration::from_millis(100), rx.recv())
14304 .await
14305 .is_err()
14306 );
14307 let pending = client.dispatch.pending_sendtx.lock().clone();
14308 assert_eq!(pending.len(), 2);
14309 assert_eq!(pending[0].batch_id, pending[1].batch_id);
14310 assert!(
14311 pending[0]
14312 .batch_id
14313 .as_ref()
14314 .unwrap()
14315 .starts_with("tx-batch:")
14316 );
14317 assert_eq!(pending[0].nonce, TEST_NEXT_NONCE);
14318 assert_eq!(pending[1].nonce, TEST_NEXT_NONCE + 1);
14319
14320 for cancel in cancels {
14321 assert_eq!(
14322 client
14323 .dispatch
14324 .pending_order_action(&cancel.client_order_id),
14325 Some(PendingOrderAction::Cancel)
14326 );
14327 }
14328
14329 assert_eq!(
14330 client
14331 .dispatch
14332 .nonce_manager
14333 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
14334 Some(TEST_NEXT_NONCE + 1)
14335 );
14336 return;
14337 }
14338
14339 let first = recv_order_event(&mut rx).await;
14340 let second = recv_order_event(&mut rx).await;
14341 let rejected_ids = [first, second].map(|event| match event {
14342 OrderEventAny::CancelRejected(e) => {
14343 assert!(e.reason.contains("Lighter cancel_order dispatch failed"),);
14344 e.client_order_id
14345 }
14346 other => panic!("expected CancelRejected, was {other:?}"),
14347 });
14348
14349 for cancel in cancels {
14350 assert!(rejected_ids.contains(&cancel.client_order_id));
14351 assert_eq!(
14352 client
14353 .dispatch
14354 .pending_order_action(&cancel.client_order_id),
14355 None,
14356 );
14357 }
14358 assert_nonce_reusable(&client.dispatch);
14359 assert_eq!(
14360 client.dispatch.pending_sendtx_len(),
14361 0,
14362 "local-send-failure must remove batch cancel pending entries",
14363 );
14364 }
14365
14366 #[rstest]
14367 #[case::spill(16)]
14368 #[case::three(31)]
14369 #[tokio::test]
14370 async fn batch_cancel_orders_chunks_large_commands_on_send_failure(#[case] count: usize) {
14371 let (client, cache, mut rx) = create_execution_client();
14372 let instrument_id = register_test_instrument(&client, &cache);
14373 let mut factory = test_order_factory();
14374
14375 let cancels = (0..count)
14376 .map(|i| {
14377 let order = test_limit_order(
14378 &mut factory,
14379 instrument_id,
14380 &format!("O-BATCH-CANCEL-MAX-{i}"),
14381 );
14382 let id = order.client_order_id();
14383 let venue = VenueOrderId::from(format!("{}", 1_000 + i));
14384 cache_pending_cancel_order(&cache, order, venue);
14385
14386 CancelOrder::new(
14387 trader_id(),
14388 Some(client_id()),
14389 strategy_id(),
14390 instrument_id,
14391 id,
14392 Some(venue),
14393 UUID4::new(),
14394 UnixNanos::default(),
14395 None,
14396 None,
14397 )
14398 })
14399 .collect::<Vec<_>>();
14400
14401 let command = BatchCancelOrders::new(
14402 trader_id(),
14403 Some(client_id()),
14404 strategy_id(),
14405 instrument_id,
14406 cancels.clone(),
14407 UUID4::new(),
14408 UnixNanos::default(),
14409 None,
14410 None,
14411 );
14412 client.batch_cancel_orders(command).unwrap();
14413
14414 let mut rejected = Vec::new();
14415
14416 for _ in &cancels {
14417 match recv_order_event(&mut rx).await {
14418 OrderEventAny::CancelRejected(e) => {
14419 assert!(e.reason.contains("Lighter cancel_order dispatch failed"));
14420 rejected.push((e.client_order_id, e.venue_order_id));
14421 }
14422 other => panic!("expected CancelRejected, was {other:?}"),
14423 }
14424 }
14425
14426 let mut expected: Vec<_> = cancels
14427 .iter()
14428 .map(|cancel| (cancel.client_order_id, cancel.venue_order_id))
14429 .collect();
14430 expected.sort();
14431 rejected.sort();
14432 assert_eq!(rejected, expected);
14433
14434 for cancel in &cancels {
14435 assert_eq!(
14436 client
14437 .dispatch
14438 .pending_order_action(&cancel.client_order_id),
14439 None
14440 );
14441 assert_eq!(
14442 client
14443 .dispatch
14444 .pending_cancel_nonce(&cancel.client_order_id),
14445 None
14446 );
14447 }
14448
14449 assert!(
14450 tokio::time::timeout(Duration::from_millis(50), rx.recv())
14451 .await
14452 .is_err(),
14453 );
14454 assert_nonce_reusable(&client.dispatch);
14455 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
14456 }
14457
14458 #[rstest]
14459 #[case::success(200, "batch-a", 0, vec!["hash-1", "hash-2"], 2, false)]
14460 #[case::partial_ack(200, "batch-a", 0, vec!["hash-2"], 1, false)]
14461 #[case::missing_hashes(200, "batch-a", 0, vec![], 0, false)]
14462 #[case::wrong_hashes(200, "batch-a", 0, vec!["unknown"], 0, false)]
14463 #[case::wrong_id(200, "unknown", 0, vec!["hash-1", "hash-2"], 0, false)]
14464 #[case::stale(200, "batch-a", 1, vec!["hash-1", "hash-2"], 0, false)]
14465 #[case::invalid_nonce(21104, "batch-a", 0, vec![], 0, true)]
14466 #[case::rejected(21727, "batch-a", 0, vec![], 0, false)]
14467 #[tokio::test]
14468 async fn batch_cancel_response_correlates_members_and_preserves_nonces(
14469 #[case] code: i64,
14470 #[case] id: &str,
14471 #[case] epoch: u64,
14472 #[case] hashes: Vec<&str>,
14473 #[case] acked_count: usize,
14474 #[case] resync_expected: bool,
14475 ) {
14476 let (client, cache, mut rx) = create_execution_client();
14477 let instrument_id = register_test_instrument(&client, &cache);
14478 for i in 0..3 {
14479 let nonce = client
14480 .dispatch
14481 .nonce_manager
14482 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
14483 .unwrap();
14484 let client_order_id = ClientOrderId::from(format!("O-BATCH-{i}"));
14485 client
14486 .dispatch
14487 .set_pending_order_action(client_order_id, PendingOrderAction::Cancel);
14488 client
14489 .dispatch
14490 .set_pending_cancel_nonce(&client_order_id, 0, nonce, true);
14491 client.dispatch.enqueue_pending_sendtx(PendingSendTx {
14492 batch_id: Some(if i == 0 { "batch-b" } else { "batch-a" }.to_string()),
14493 connection_epoch: 0,
14494 kind: PendingSendTxKind::Cancel {
14495 emit_cancel_rejected: true,
14496 strategy_id: strategy_id(),
14497 instrument_id,
14498 client_order_id,
14499 venue_order_id: Some(VenueOrderId::from(format!("{}", 100 + i))),
14500 },
14501 submitted_at: UnixNanos::default(),
14502 nonce,
14503 api_key_index: TEST_API_KEY_INDEX,
14504 tx_hash: format!("hash-{i}"),
14505 });
14506 }
14507
14508 assert!(client.dispatch.pop_pending_sendtx_if_only(0).is_none());
14509 let hashes: Vec<_> = hashes.into_iter().map(str::to_string).collect();
14510 let (acked, resync) = handle_send_tx_batch_result(
14511 &client.dispatch,
14512 &client.emitter,
14513 Some(TEST_ACCOUNT_INDEX_I64),
14514 epoch,
14515 UnixNanos::default(),
14516 id,
14517 code,
14518 "batch failure",
14519 &hashes,
14520 );
14521 assert_eq!(resync, resync_expected);
14522 assert_eq!(acked.len(), acked_count);
14523 let rejected = code != 200;
14524 assert_eq!(
14525 client.dispatch.pending_sendtx_len(),
14526 if rejected { 1 } else { 3 - acked_count }
14527 );
14528
14529 if rejected {
14530 for i in [2, 1] {
14531 match recv_order_event(&mut rx).await {
14532 OrderEventAny::CancelRejected(event) => {
14533 assert_eq!(
14534 event.client_order_id,
14535 ClientOrderId::from(format!("O-BATCH-{i}"))
14536 );
14537 assert_eq!(event.strategy_id, strategy_id());
14538 assert_eq!(event.instrument_id, instrument_id);
14539 assert_eq!(
14540 event.venue_order_id,
14541 Some(VenueOrderId::from(format!("{}", 100 + i)))
14542 );
14543 }
14544 event => panic!("expected rejection, was {event:?}"),
14545 }
14546
14547 assert_eq!(
14548 client
14549 .dispatch
14550 .pending_order_action(&ClientOrderId::from(format!("O-BATCH-{i}"))),
14551 None
14552 );
14553 }
14554
14555 assert_eq!(
14556 client
14557 .dispatch
14558 .nonce_manager
14559 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
14560 Some(TEST_NEXT_NONCE)
14561 );
14562 } else {
14563 for pending in &acked {
14564 assert!(hashes.contains(&pending.tx_hash));
14565 }
14566
14567 assert_eq!(
14568 client
14569 .dispatch
14570 .nonce_manager
14571 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
14572 Some(TEST_NEXT_NONCE + 2)
14573 );
14574 }
14575
14576 assert_eq!(
14577 client
14578 .dispatch
14579 .pending_order_action(&ClientOrderId::from("O-BATCH-0")),
14580 Some(PendingOrderAction::Cancel)
14581 );
14582 assert!(
14583 tokio::time::timeout(Duration::from_millis(20), rx.recv())
14584 .await
14585 .is_err()
14586 );
14587 let remaining = client.dispatch.drain_pending_sendtx(0);
14588 assert_eq!(remaining.len(), if rejected { 1 } else { 3 - acked_count });
14589 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
14590 }
14591
14592 #[rstest]
14593 fn cancel_order_from_cancel_all_preserves_tracing_ids() {
14594 let instrument_id = InstrumentId::from("ETH-PERP.LIGHTER");
14595 let client_order_id = ClientOrderId::from("O-CANCEL-ALL-CHILD");
14596 let command_id = UUID4::new();
14597 let correlation_id = UUID4::new();
14598 let causation_id = UUID4::new();
14599 let ts_init = UnixNanos::default();
14600 let mut cmd = CancelAllOrders::new(
14601 trader_id(),
14602 Some(client_id()),
14603 strategy_id(),
14604 instrument_id,
14605 Some(OrderSide::Buy),
14606 command_id,
14607 ts_init,
14608 None,
14609 Some(correlation_id),
14610 );
14611 cmd.causation_id = Some(causation_id);
14612
14613 let owner = StrategyId::from("S-002");
14614 let order_cmd = cancel_order_from_cancel_all(&cmd, client_order_id, owner);
14615
14616 assert_eq!(order_cmd.trader_id, trader_id());
14617 assert_eq!(order_cmd.client_id, Some(client_id()));
14618 assert_eq!(order_cmd.strategy_id, owner);
14619 assert_eq!(order_cmd.instrument_id, instrument_id);
14620 assert_eq!(order_cmd.client_order_id, client_order_id);
14621 assert_eq!(order_cmd.venue_order_id, None);
14622 assert_eq!(order_cmd.command_id, command_id);
14623 assert_eq!(order_cmd.ts_init, ts_init);
14624 assert_eq!(order_cmd.params, None);
14625 assert_eq!(order_cmd.correlation_id, Some(correlation_id));
14626 assert_eq!(order_cmd.causation_id, Some(causation_id));
14627 }
14628
14629 #[rstest]
14630 #[case::not_sent(false)]
14631 #[case::ambiguous(true)]
14632 #[tokio::test]
14633 async fn batch_modify_orders_send_failure_preserves_delivery_semantics(
14634 #[case] ambiguous: bool,
14635 ) {
14636 let (client, cache, mut rx) = create_execution_client();
14637 let instrument_id = register_test_instrument(&client, &cache);
14638 let mut factory = test_order_factory();
14639
14640 let modifies: Vec<_> = (0..2)
14641 .map(|i| {
14642 let order =
14643 test_limit_order(&mut factory, instrument_id, &format!("O-BATCH-MODIFY-{i}"));
14644 let id = order.client_order_id();
14645 cache_order(&cache, order);
14646
14647 ModifyOrder::new(
14648 trader_id(),
14649 Some(client_id()),
14650 strategy_id(),
14651 instrument_id,
14652 id,
14653 Some(VenueOrderId::from(format!("{}", 123 + i))),
14654 Some(Quantity::from("0.2000")),
14655 Some(Price::from("2362.00")),
14656 None,
14657 UUID4::new(),
14658 UnixNanos::default(),
14659 None,
14660 None,
14661 )
14662 })
14663 .collect();
14664
14665 if ambiguous {
14666 client.ws_client.drop_next_send_tx_result_for_test().await;
14667 }
14668
14669 client
14670 .batch_modify_orders(BatchModifyOrders::new(
14671 trader_id(),
14672 Some(client_id()),
14673 strategy_id(),
14674 instrument_id,
14675 modifies.clone(),
14676 UUID4::new(),
14677 UnixNanos::default(),
14678 None,
14679 None,
14680 ))
14681 .unwrap();
14682
14683 if ambiguous {
14684 wait_until_async(
14685 || async { client.dispatch.pending_sendtx_len() == 2 },
14686 Duration::from_secs(2),
14687 )
14688 .await;
14689 assert!(
14690 tokio::time::timeout(Duration::from_millis(100), rx.recv())
14691 .await
14692 .is_err()
14693 );
14694 let pending = client.dispatch.pending_sendtx.lock().clone();
14695 assert_eq!(pending.len(), 2);
14696 assert_eq!(pending[0].batch_id, pending[1].batch_id);
14697 assert!(
14698 pending[0]
14699 .batch_id
14700 .as_ref()
14701 .unwrap()
14702 .starts_with(TX_BATCH_ID_PREFIX)
14703 );
14704 assert_eq!(pending[0].nonce, TEST_NEXT_NONCE);
14705 assert_eq!(pending[1].nonce, TEST_NEXT_NONCE + 1);
14706
14707 for modify in &modifies {
14708 assert_eq!(
14709 client
14710 .dispatch
14711 .pending_order_action(&modify.client_order_id),
14712 Some(PendingOrderAction::Modify)
14713 );
14714 }
14715 } else {
14716 let mut ids = Vec::new();
14717
14718 for _ in 0..2 {
14719 match recv_order_event(&mut rx).await {
14720 OrderEventAny::ModifyRejected(e) => {
14721 assert!(e.reason.contains("Lighter modify_order dispatch failed"));
14722 assert_eq!(e.instrument_id, instrument_id);
14723 assert_eq!(e.strategy_id, strategy_id());
14724 ids.push((e.client_order_id, e.venue_order_id));
14725 }
14726 other => panic!("expected ModifyRejected, was {other:?}"),
14727 }
14728 }
14729
14730 ids.sort();
14731 let mut expected: Vec<_> = modifies
14732 .iter()
14733 .map(|modify| (modify.client_order_id, modify.venue_order_id))
14734 .collect();
14735 expected.sort();
14736 assert_eq!(ids, expected);
14737 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
14738
14739 for modify in &modifies {
14740 assert_eq!(
14741 client
14742 .dispatch
14743 .pending_order_action(&modify.client_order_id),
14744 None
14745 );
14746 }
14747
14748 assert_nonce_reusable(&client.dispatch);
14749 }
14750 }
14751
14752 #[tokio::test]
14753 async fn batch_modify_orders_rejects_invalid_child_and_dispatches_supported() {
14754 let (client, cache, mut rx) = create_execution_client();
14755 let instrument_id = register_test_instrument(&client, &cache);
14756 let mut factory = test_order_factory();
14757 let mut modifies = Vec::new();
14758
14759 for (id, venue_id) in [("O-MODIFY-INVALID", "invalid"), ("O-MODIFY-VALID", "456")] {
14760 let order = test_limit_order(&mut factory, instrument_id, id);
14761 let client_order_id = order.client_order_id();
14762 cache_order(&cache, order);
14763 modifies.push(ModifyOrder::new(
14764 trader_id(),
14765 Some(client_id()),
14766 strategy_id(),
14767 instrument_id,
14768 client_order_id,
14769 Some(VenueOrderId::from(venue_id)),
14770 Some(Quantity::from("0.2000")),
14771 Some(Price::from("2362.00")),
14772 None,
14773 UUID4::new(),
14774 UnixNanos::default(),
14775 None,
14776 None,
14777 ));
14778 }
14779
14780 client.ws_client.drop_next_send_tx_result_for_test().await;
14781 client
14782 .batch_modify_orders(BatchModifyOrders::new(
14783 trader_id(),
14784 Some(client_id()),
14785 strategy_id(),
14786 instrument_id,
14787 modifies.clone(),
14788 UUID4::new(),
14789 UnixNanos::default(),
14790 None,
14791 None,
14792 ))
14793 .unwrap();
14794 let event = recv_order_event(&mut rx).await;
14795
14796 let OrderEventAny::ModifyRejected(event) = event else {
14797 panic!("expected invalid-child rejection")
14798 };
14799
14800 assert_eq!(event.client_order_id, modifies[0].client_order_id);
14801 assert_eq!(event.venue_order_id, modifies[0].venue_order_id);
14802 assert!(event.reason.contains("invalid"));
14803 wait_until_async(
14804 || async { client.dispatch.pending_sendtx_len() == 1 },
14805 Duration::from_secs(2),
14806 )
14807 .await;
14808 let pending = client.dispatch.pending_sendtx.lock().clone();
14809 assert_eq!(pending.len(), 1);
14810 assert_eq!(pending[0].nonce, TEST_NEXT_NONCE);
14811 assert!(
14812 matches!(&pending[0].kind, PendingSendTxKind::Modify { client_order_id, .. } if *client_order_id == modifies[1].client_order_id)
14813 );
14814 assert_eq!(
14815 client
14816 .dispatch
14817 .pending_order_action(&modifies[0].client_order_id),
14818 None
14819 );
14820 assert_eq!(
14821 client
14822 .dispatch
14823 .pending_order_action(&modifies[1].client_order_id),
14824 Some(PendingOrderAction::Modify)
14825 );
14826 assert!(rx.try_recv().is_err());
14827 }
14828
14829 #[rstest]
14830 #[case::empty(0)]
14831 #[case::full(15)]
14832 #[case::spill(16)]
14833 #[case::three(31)]
14834 #[tokio::test]
14835 async fn batch_modify_orders_chunks_large_commands_on_send_failure(#[case] count: usize) {
14836 let (client, cache, mut rx) = create_execution_client();
14837 let instrument_id = register_test_instrument(&client, &cache);
14838 let mut factory = test_order_factory();
14839
14840 let modifies: Vec<_> = (0..count)
14841 .map(|i| {
14842 let order =
14843 test_limit_order(&mut factory, instrument_id, &format!("O-BATCH-SIZE-{i}"));
14844 let id = order.client_order_id();
14845 cache_order(&cache, order);
14846
14847 ModifyOrder::new(
14848 trader_id(),
14849 Some(client_id()),
14850 strategy_id(),
14851 instrument_id,
14852 id,
14853 Some(VenueOrderId::from(format!("{}", 123 + i))),
14854 Some(Quantity::from("0.2000")),
14855 Some(Price::from("2362.00")),
14856 None,
14857 UUID4::new(),
14858 UnixNanos::default(),
14859 None,
14860 None,
14861 )
14862 })
14863 .collect();
14864
14865 client
14866 .batch_modify_orders(BatchModifyOrders::new(
14867 trader_id(),
14868 Some(client_id()),
14869 strategy_id(),
14870 instrument_id,
14871 modifies.clone(),
14872 UUID4::new(),
14873 UnixNanos::default(),
14874 None,
14875 None,
14876 ))
14877 .unwrap();
14878
14879 let mut rejected = Vec::new();
14880
14881 for _ in &modifies {
14882 match recv_order_event(&mut rx).await {
14883 OrderEventAny::ModifyRejected(e) => {
14884 assert!(e.reason.contains("Lighter modify_order dispatch failed"));
14885 rejected.push((e.client_order_id, e.venue_order_id));
14886 }
14887 other => panic!("expected ModifyRejected, was {other:?}"),
14888 }
14889 }
14890
14891 let mut expected: Vec<_> = modifies
14892 .iter()
14893 .map(|modify| (modify.client_order_id, modify.venue_order_id))
14894 .collect();
14895 expected.sort();
14896 rejected.sort();
14897 assert_eq!(rejected, expected);
14898
14899 for modify in &modifies {
14900 assert_eq!(
14901 client
14902 .dispatch
14903 .pending_order_action(&modify.client_order_id),
14904 None
14905 );
14906 assert_eq!(
14907 client
14908 .dispatch
14909 .pending_cancel_nonce(&modify.client_order_id),
14910 None
14911 );
14912 }
14913
14914 assert!(
14915 tokio::time::timeout(Duration::from_millis(50), rx.recv())
14916 .await
14917 .is_err(),
14918 );
14919 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
14920 assert_nonce_reusable(&client.dispatch);
14921 }
14922
14923 #[tokio::test]
14924 async fn modify_order_send_failure_emits_modify_rejected_and_rolls_back() {
14925 let (client, cache, mut rx) = create_execution_client();
14926 let instrument_id = register_test_instrument(&client, &cache);
14927 let mut factory = test_order_factory();
14928 let order = test_limit_order(&mut factory, instrument_id, "O-MODIFY-FAIL");
14929 let client_order_id = order.client_order_id();
14930 let venue_order_id = VenueOrderId::from("123");
14931 cache_order(&cache, order);
14932
14933 let command = ModifyOrder::new(
14934 trader_id(),
14935 Some(client_id()),
14936 strategy_id(),
14937 instrument_id,
14938 client_order_id,
14939 Some(venue_order_id),
14940 Some(Quantity::from("0.2000")),
14941 Some(Price::from("2362.00")),
14942 None,
14943 UUID4::new(),
14944 UnixNanos::default(),
14945 None,
14946 None,
14947 );
14948 client.modify_order(command).unwrap();
14949
14950 let rejected = recv_order_event(&mut rx).await;
14951
14952 match rejected {
14953 OrderEventAny::ModifyRejected(event) => {
14954 assert_eq!(event.client_order_id, client_order_id);
14955 assert_eq!(event.instrument_id, instrument_id);
14956 assert_eq!(event.venue_order_id, Some(venue_order_id));
14957 assert!(
14958 event
14959 .reason
14960 .contains("Lighter modify_order dispatch failed"),
14961 );
14962 assert!(event.reason.contains("handler unavailable"));
14963 }
14964 event => panic!("expected modify rejected event, was {event:?}"),
14965 }
14966
14967 assert_nonce_reusable(&client.dispatch);
14968 assert_eq!(
14969 client.dispatch.pending_sendtx_len(),
14970 0,
14971 "local-send-failure must remove the pending modify entry",
14972 );
14973 assert_eq!(client.dispatch.pending_order_action(&client_order_id), None);
14974 }
14975
14976 #[tokio::test]
14977 async fn modify_order_unknown_send_outcome_retains_pending_state() {
14978 let (client, cache, mut rx) = create_execution_client();
14979 let instrument_id = register_test_instrument(&client, &cache);
14980 let mut factory = test_order_factory();
14981 let order = test_limit_order(&mut factory, instrument_id, "O-MODIFY-UNKNOWN");
14982 let client_order_id = order.client_order_id();
14983 let venue_order_id = VenueOrderId::from("123");
14984 cache_order(&cache, order);
14985 client.ws_client.drop_next_send_tx_result_for_test().await;
14986
14987 client
14988 .modify_order(ModifyOrder::new(
14989 trader_id(),
14990 Some(client_id()),
14991 strategy_id(),
14992 instrument_id,
14993 client_order_id,
14994 Some(venue_order_id),
14995 Some(Quantity::from("0.2000")),
14996 Some(Price::from("2362.00")),
14997 None,
14998 UUID4::new(),
14999 UnixNanos::default(),
15000 None,
15001 None,
15002 ))
15003 .unwrap();
15004
15005 assert!(
15006 tokio::time::timeout(Duration::from_millis(50), rx.recv())
15007 .await
15008 .is_err(),
15009 "unknown outcome must not emit OrderModifyRejected",
15010 );
15011 assert_eq!(client.dispatch.pending_sendtx_len(), 1);
15012 assert_eq!(
15013 client.dispatch.pending_order_action(&client_order_id),
15014 Some(PendingOrderAction::Modify),
15015 );
15016 assert_eq!(
15017 client
15018 .dispatch
15019 .nonce_manager
15020 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
15021 Some(TEST_NEXT_NONCE),
15022 );
15023 }
15024
15025 #[tokio::test]
15026 async fn modify_order_prepare_failure_emits_modify_rejected_without_dispatch() {
15027 let (client, cache, mut rx) = create_execution_client();
15028 let instrument_id = register_test_instrument(&client, &cache);
15029 let client_order_id = ClientOrderId::from("O-MODIFY-NO-CACHE");
15030 let venue_order_id = VenueOrderId::from("123");
15031
15032 let command = ModifyOrder::new(
15033 trader_id(),
15034 Some(client_id()),
15035 strategy_id(),
15036 instrument_id,
15037 client_order_id,
15038 Some(venue_order_id),
15039 Some(Quantity::from("0.2000")),
15040 Some(Price::from("2362.00")),
15041 None,
15042 UUID4::new(),
15043 UnixNanos::default(),
15044 None,
15045 None,
15046 );
15047 client.modify_order(command).unwrap();
15048
15049 let rejected = recv_order_event(&mut rx).await;
15050 match rejected {
15051 OrderEventAny::ModifyRejected(event) => {
15052 assert_eq!(event.client_order_id, client_order_id);
15053 assert_eq!(event.instrument_id, instrument_id);
15054 assert_eq!(event.venue_order_id, Some(venue_order_id));
15055 assert!(event.reason.contains("Lighter modify_order failed"));
15056 assert!(event.reason.contains("order not found in cache"));
15057 }
15058 event => panic!("expected modify rejected event, was {event:?}"),
15059 }
15060
15061 assert_nonce_reusable(&client.dispatch);
15062 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
15063 assert_eq!(client.dispatch.pending_order_action(&client_order_id), None);
15064 assert!(
15065 tokio::time::timeout(Duration::from_millis(50), rx.recv())
15066 .await
15067 .is_err(),
15068 "prepare failure must emit exactly one modify rejection",
15069 );
15070 }
15071
15072 #[tokio::test]
15073 async fn modify_order_prepare_failure_emits_modify_rejected_when_instrument_uncached() {
15074 let (client, cache, mut rx) = create_execution_client();
15075 let instrument_id =
15076 client
15077 .registry
15078 .insert(TEST_MARKET_INDEX, "ETH", LighterProductType::Perp);
15079 let mut factory = test_order_factory();
15080 let order = test_limit_order(&mut factory, instrument_id, "O-MODIFY-NO-INSTRUMENT");
15081 let client_order_id = order.client_order_id();
15082 let venue_order_id = VenueOrderId::from("123");
15083 cache_order(&cache, order);
15084
15085 let command = ModifyOrder::new(
15086 trader_id(),
15087 Some(client_id()),
15088 strategy_id(),
15089 instrument_id,
15090 client_order_id,
15091 Some(venue_order_id),
15092 Some(Quantity::from("0.2000")),
15093 Some(Price::from("2362.00")),
15094 None,
15095 UUID4::new(),
15096 UnixNanos::default(),
15097 None,
15098 None,
15099 );
15100 client.modify_order(command).unwrap();
15101
15102 let rejected = recv_order_event(&mut rx).await;
15103 match rejected {
15104 OrderEventAny::ModifyRejected(event) => {
15105 assert_eq!(event.client_order_id, client_order_id);
15106 assert_eq!(event.instrument_id, instrument_id);
15107 assert_eq!(event.venue_order_id, Some(venue_order_id));
15108 assert!(event.reason.contains("Lighter modify_order failed"));
15109 assert!(event.reason.contains("instrument not found in cache"));
15110 }
15111 event => panic!("expected modify rejected event, was {event:?}"),
15112 }
15113
15114 assert_nonce_reusable(&client.dispatch);
15115 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
15116 assert!(
15117 tokio::time::timeout(Duration::from_millis(50), rx.recv())
15118 .await
15119 .is_err(),
15120 "prepare failure must emit exactly one modify rejection",
15121 );
15122 }
15123
15124 #[tokio::test]
15125 async fn modify_stop_market_derives_price_from_trigger_without_explicit_price() {
15126 let (client, cache, mut rx) = create_execution_client();
15131 let instrument_id = register_test_instrument(&client, &cache);
15132 let mut factory = test_order_factory();
15133 let order = factory.stop_market(
15134 instrument_id,
15135 OrderSide::Sell,
15136 Quantity::from("0.1000"),
15137 Price::from("2300.00"), None,
15139 Some(TimeInForce::Gtc),
15140 None,
15141 Some(false),
15142 Some(false),
15143 None,
15144 None,
15145 None,
15146 None,
15147 None,
15148 None,
15149 Some(ClientOrderId::from("O-MODIFY-STOP-MARKET")),
15150 );
15151 let client_order_id = order.client_order_id();
15152 let venue_order_id = VenueOrderId::from("123");
15153 cache_order(&cache, order);
15154
15155 let command = ModifyOrder::new(
15156 trader_id(),
15157 Some(client_id()),
15158 strategy_id(),
15159 instrument_id,
15160 client_order_id,
15161 Some(venue_order_id),
15162 None,
15163 None,
15164 Some(Price::from("2310.00")),
15165 UUID4::new(),
15166 UnixNanos::default(),
15167 None,
15168 None,
15169 );
15170 client.modify_order(command).unwrap();
15171
15172 let rejected = recv_order_event(&mut rx).await;
15173 match rejected {
15174 OrderEventAny::ModifyRejected(event) => {
15175 assert!(
15176 !event.reason.contains("requires a price"),
15177 "trigger-only stop modify must not trip the price guard, was: {}",
15178 event.reason,
15179 );
15180 assert!(
15181 event.reason.contains("dispatch failed"),
15182 "expected send-stage failure after a successful prepare, was: {}",
15183 event.reason,
15184 );
15185 }
15186 event => panic!("expected modify rejected event, was {event:?}"),
15187 }
15188 assert_nonce_reusable(&client.dispatch);
15189 }
15190
15191 #[tokio::test]
15192 async fn modify_order_rejects_sub_tick_quantity_before_nonce_allocation() {
15193 let (client, cache, mut rx) = create_execution_client();
15194 let instrument_id = register_test_instrument(&client, &cache);
15195 let mut factory = test_order_factory();
15196 let order = test_limit_order(&mut factory, instrument_id, "O-MODIFY-SUB-TICK");
15197 let client_order_id = order.client_order_id();
15198 cache_order(&cache, order);
15199
15200 client
15201 .modify_order(ModifyOrder::new(
15202 trader_id(),
15203 Some(client_id()),
15204 strategy_id(),
15205 instrument_id,
15206 client_order_id,
15207 Some(VenueOrderId::from("123")),
15208 Some(Quantity::from("0.00001")),
15209 Some(Price::from("2362.00")),
15210 None,
15211 UUID4::new(),
15212 UnixNanos::default(),
15213 None,
15214 None,
15215 ))
15216 .unwrap();
15217
15218 assert_modify_rejected_reason(&mut rx, "rounds to 0 ticks").await;
15219 assert_nonce_reusable(&client.dispatch);
15220 }
15221
15222 #[tokio::test]
15223 async fn modify_order_rejects_below_min_notional_before_nonce_allocation() {
15224 let (client, cache, mut rx) = create_execution_client();
15225 let instrument_id = register_test_instrument(&client, &cache);
15226 let mut factory = test_order_factory();
15227 let order = test_limit_order(&mut factory, instrument_id, "O-MODIFY-MIN-NOTIONAL");
15228 let client_order_id = order.client_order_id();
15229 cache_order(&cache, order);
15230
15231 client
15232 .modify_order(ModifyOrder::new(
15233 trader_id(),
15234 Some(client_id()),
15235 strategy_id(),
15236 instrument_id,
15237 client_order_id,
15238 Some(VenueOrderId::from("123")),
15239 Some(Quantity::from("0.0001")),
15240 Some(Price::from("2362.00")),
15241 None,
15242 UUID4::new(),
15243 UnixNanos::default(),
15244 None,
15245 None,
15246 ))
15247 .unwrap();
15248
15249 assert_modify_rejected_reason(&mut rx, "below Lighter min_quote_amount").await;
15250 assert_nonce_reusable(&client.dispatch);
15251 }
15252
15253 #[tokio::test]
15254 async fn modify_conditional_order_rejects_sub_tick_trigger_before_nonce_allocation() {
15255 let (client, cache, mut rx) = create_execution_client();
15256 let instrument_id = register_test_instrument(&client, &cache);
15257 let mut factory = test_order_factory();
15258 let order = factory.stop_market(
15259 instrument_id,
15260 OrderSide::Sell,
15261 Quantity::from("0.1000"),
15262 Price::from("2300.00"),
15263 None,
15264 Some(TimeInForce::Gtc),
15265 None,
15266 Some(false),
15267 Some(false),
15268 None,
15269 None,
15270 None,
15271 None,
15272 None,
15273 None,
15274 Some(ClientOrderId::from("O-MODIFY-SUB-TICK-TRIGGER")),
15275 );
15276 let client_order_id = order.client_order_id();
15277 cache_order(&cache, order);
15278
15279 client
15280 .modify_order(ModifyOrder::new(
15281 trader_id(),
15282 Some(client_id()),
15283 strategy_id(),
15284 instrument_id,
15285 client_order_id,
15286 Some(VenueOrderId::from("123")),
15287 None,
15288 None,
15289 Some(Price::from("0.001")),
15290 UUID4::new(),
15291 UnixNanos::default(),
15292 None,
15293 None,
15294 ))
15295 .unwrap();
15296
15297 assert_modify_rejected_reason(&mut rx, "rounds to 0 ticks").await;
15298 assert_nonce_reusable(&client.dispatch);
15299 }
15300
15301 #[tokio::test]
15302 async fn modify_order_nonce_prepare_failure_emits_modify_rejected() {
15303 let mut config = test_config();
15304 config.base_url_http = Some(spawn_next_nonce_server(101).await);
15305 let (client, cache, mut rx) = create_execution_client_with_config(config);
15306 let instrument_id = register_test_instrument(&client, &cache);
15307 let mut factory = test_order_factory();
15308 let order = test_limit_order(&mut factory, instrument_id, "O-MODIFY-NONCE-FAIL");
15309 let client_order_id = order.client_order_id();
15310 let venue_order_id = VenueOrderId::from("123");
15311 cache_order(&cache, order);
15312
15313 let window = i64::from(client.dispatch.nonce_manager.skip_window());
15314 for _ in 0..window {
15315 client
15316 .dispatch
15317 .nonce_manager
15318 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
15319 .unwrap();
15320 }
15321
15322 let command = ModifyOrder::new(
15323 trader_id(),
15324 Some(client_id()),
15325 strategy_id(),
15326 instrument_id,
15327 client_order_id,
15328 Some(venue_order_id),
15329 Some(Quantity::from("0.2000")),
15330 Some(Price::from("2362.00")),
15331 None,
15332 UUID4::new(),
15333 UnixNanos::default(),
15334 None,
15335 None,
15336 );
15337 client.modify_order(command).unwrap();
15338
15339 let rejected = recv_order_event(&mut rx).await;
15340 match rejected {
15341 OrderEventAny::ModifyRejected(event) => {
15342 assert_eq!(event.client_order_id, client_order_id);
15343 assert_eq!(event.venue_order_id, Some(venue_order_id));
15344 assert!(event.reason.contains("failed to allocate Lighter nonce"),);
15345 assert!(event.reason.contains("skip-window exhausted"));
15346 }
15347 event => panic!("expected modify rejected event, was {event:?}"),
15348 }
15349
15350 wait_for_spawned_tasks(&client).await;
15351 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
15352 assert_eq!(
15353 client
15354 .dispatch
15355 .nonce_manager
15356 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
15357 .unwrap(),
15358 101,
15359 );
15360 }
15361
15362 #[tokio::test]
15363 async fn update_leverage_requires_credentials() {
15364 let (mut client, _cache, _rx) = create_execution_client();
15365 client.credential = None;
15366 let instrument_id = InstrumentId::from("ETH-PERP.LIGHTER");
15367
15368 let err = client
15369 .update_leverage(instrument_id, 500, LighterPositionMarginMode::Isolated)
15370 .unwrap_err();
15371
15372 assert!(
15373 err.to_string()
15374 .contains("cannot update leverage without credentials"),
15375 );
15376 }
15377
15378 #[tokio::test]
15379 async fn update_leverage_requires_registered_instrument() {
15380 let (client, _cache, _rx) = create_execution_client();
15381 let unknown = InstrumentId::from("DOGE-PERP.LIGHTER");
15382
15383 let err = client
15384 .update_leverage(unknown, 500, LighterPositionMarginMode::Isolated)
15385 .unwrap_err();
15386
15387 assert!(
15388 err.to_string()
15389 .contains("no Lighter market_index registered")
15390 );
15391 assert_nonce_reusable(&client.dispatch);
15394 }
15395
15396 #[tokio::test]
15397 async fn update_leverage_dispatches_and_rolls_back_on_send_failure() {
15398 let (client, cache, _rx) = create_execution_client();
15399 let instrument_id = register_test_instrument(&client, &cache);
15400
15401 client
15402 .update_leverage(instrument_id, 500, LighterPositionMarginMode::Isolated)
15403 .unwrap();
15404
15405 wait_for_spawned_tasks(&client).await;
15406 assert_nonce_reusable(&client.dispatch);
15407 }
15408
15409 #[tokio::test]
15410 async fn update_leverage_unknown_send_outcome_retains_pending_nonce() {
15411 let (client, cache, _rx) = create_execution_client();
15412 let instrument_id = register_test_instrument(&client, &cache);
15413 client.ws_client.drop_next_send_tx_result_for_test().await;
15414
15415 client
15416 .update_leverage(instrument_id, 500, LighterPositionMarginMode::Isolated)
15417 .unwrap();
15418
15419 wait_for_spawned_tasks(&client).await;
15420 assert_eq!(client.dispatch.pending_sendtx_len(), 1);
15421 assert_eq!(
15422 client
15423 .dispatch
15424 .nonce_manager
15425 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
15426 Some(TEST_NEXT_NONCE),
15427 );
15428 }
15429
15430 #[tokio::test]
15431 async fn update_leverage_rejects_zero_margin_fraction() {
15432 let (client, cache, _rx) = create_execution_client();
15433 let instrument_id = register_test_instrument(&client, &cache);
15434
15435 let err = client
15436 .update_leverage(instrument_id, 0, LighterPositionMarginMode::Cross)
15437 .unwrap_err();
15438 assert!(err.to_string().contains("must be in 1..=10_000"));
15439 }
15440
15441 #[tokio::test]
15442 async fn update_leverage_rejects_above_margin_fraction_tick() {
15443 let (client, cache, _rx) = create_execution_client();
15444 let instrument_id = register_test_instrument(&client, &cache);
15445
15446 let err = client
15447 .update_leverage(instrument_id, 10_001, LighterPositionMarginMode::Cross)
15448 .unwrap_err();
15449 assert!(err.to_string().contains("must be in 1..=10_000"));
15450 }
15451
15452 #[tokio::test]
15453 async fn update_leverage_accepts_minimum_margin_fraction() {
15454 let (client, cache, _rx) = create_execution_client();
15457 let instrument_id = register_test_instrument(&client, &cache);
15458
15459 client
15460 .update_leverage(instrument_id, 1, LighterPositionMarginMode::Cross)
15461 .unwrap();
15462
15463 wait_for_spawned_tasks(&client).await;
15464 assert_nonce_reusable(&client.dispatch);
15465 }
15466
15467 #[tokio::test]
15468 async fn update_leverage_accepts_maximum_margin_fraction() {
15469 let (client, cache, _rx) = create_execution_client();
15472 let instrument_id = register_test_instrument(&client, &cache);
15473
15474 client
15475 .update_leverage(instrument_id, 10_000, LighterPositionMarginMode::Isolated)
15476 .unwrap();
15477
15478 wait_for_spawned_tasks(&client).await;
15479 assert_nonce_reusable(&client.dispatch);
15480 }
15481
15482 async fn wait_for_spawned_tasks(client: &LighterExecutionClient) {
15483 wait_until_async(
15484 || async { client.pending_tasks_all_finished() },
15485 Duration::from_secs(2),
15486 )
15487 .await;
15488 }
15489
15490 fn mark_all_streams_ready(client: &LighterExecutionClient) {
15491 let ready = &client.dispatch.account_streams_ready;
15492 ready.mark_orders();
15493 ready.mark_trades();
15494 ready.mark_positions();
15495 ready.mark_assets();
15496 ready.mark_user_stats();
15497 }
15498
15499 #[tokio::test]
15500 async fn await_account_streams_ready_times_out_when_no_frame_arrives() {
15501 let (client, _cache, _rx) = create_execution_client();
15504
15505 let err = client.await_account_streams_ready(0.05).await.unwrap_err();
15506
15507 assert!(
15508 err.to_string().contains("Timeout")
15509 && err.to_string().contains("Lighter account streams"),
15510 "unexpected error message, was {err}",
15511 );
15512 }
15513
15514 #[tokio::test]
15515 async fn await_account_streams_ready_returns_when_all_streams_marked() {
15516 let (client, _cache, _rx) = create_execution_client();
15517 mark_all_streams_ready(&client);
15518
15519 client.await_account_streams_ready(0.05).await.unwrap();
15520 }
15521
15522 #[tokio::test]
15523 async fn await_account_streams_ready_returns_when_streams_arrive_mid_wait() {
15524 let (client, _cache, _rx) = create_execution_client();
15525 let ready = Arc::clone(&client.dispatch.account_streams_ready);
15526 let (pending_tx, pending_rx) = tokio::sync::oneshot::channel();
15527 let mut pending_tx = Some(pending_tx);
15528 let wait = client.await_account_streams_ready(30.0);
15529 tokio::pin!(wait);
15530
15531 let producer = tokio::spawn(async move {
15532 pending_rx.await.expect("readiness waiter parked");
15533 ready.mark_orders();
15534 ready.mark_trades();
15535 ready.mark_positions();
15536 ready.mark_assets();
15537 ready.mark_user_stats();
15538 });
15539
15540 tokio::time::timeout(
15542 Duration::from_secs(2),
15543 std::future::poll_fn(|cx| {
15544 let result = wait.as_mut().poll(cx);
15545 if result.is_pending()
15546 && let Some(tx) = pending_tx.take()
15547 {
15548 tx.send(()).expect("readiness producer alive");
15549 }
15550
15551 result
15552 }),
15553 )
15554 .await
15555 .expect("marks must wake the waiter before its deadline")
15556 .unwrap();
15557
15558 assert!(pending_tx.is_none(), "waiter must initially be pending");
15559 producer.await.unwrap();
15560 }
15561
15562 #[tokio::test]
15563 async fn await_account_streams_ready_times_out_with_partial_marks() {
15564 let (client, _cache, _rx) = create_execution_client();
15568 let ready = &client.dispatch.account_streams_ready;
15569 ready.mark_orders();
15570 ready.mark_trades();
15571 ready.mark_positions();
15572
15573 let err = client.await_account_streams_ready(0.05).await.unwrap_err();
15574 assert!(
15575 err.to_string().contains("assets"),
15576 "pending list should call out the missing stream, was {err}",
15577 );
15578 }
15579
15580 #[tokio::test]
15581 async fn await_account_streams_ready_after_reset_requires_new_marks() {
15582 let (client, _cache, _rx) = create_execution_client();
15587 mark_all_streams_ready(&client);
15588 client.await_account_streams_ready(0.05).await.unwrap();
15589
15590 client.dispatch.account_streams_ready.reset();
15591
15592 let err = client.await_account_streams_ready(0.05).await.unwrap_err();
15593 let msg = err.to_string();
15594 assert!(msg.contains("orders"), "pending list missing orders: {msg}");
15595 assert!(msg.contains("trades"), "pending list missing trades: {msg}");
15596 assert!(
15597 msg.contains("positions"),
15598 "pending list missing positions: {msg}",
15599 );
15600 assert!(msg.contains("assets"), "pending list missing assets: {msg}");
15601 assert!(
15602 msg.contains("user_stats"),
15603 "pending list missing user_stats: {msg}",
15604 );
15605 }
15606
15607 fn test_market_order(
15608 factory: &mut OrderFactory,
15609 instrument_id: InstrumentId,
15610 client_order_id: &str,
15611 side: OrderSide,
15612 ) -> OrderAny {
15613 factory.market(
15614 instrument_id,
15615 side,
15616 Quantity::from("0.1000"),
15617 Some(TimeInForce::Ioc),
15618 Some(false),
15619 Some(false),
15620 None,
15621 None,
15622 None,
15623 Some(ClientOrderId::from(client_order_id)),
15624 )
15625 }
15626
15627 fn add_test_quote(
15628 cache: &Rc<RefCell<Cache>>,
15629 instrument_id: InstrumentId,
15630 bid: &str,
15631 ask: &str,
15632 ) {
15633 let quote = QuoteTick::new(
15634 instrument_id,
15635 Price::from(bid),
15636 Price::from(ask),
15637 Quantity::from("1.0000"),
15638 Quantity::from("1.0000"),
15639 UnixNanos::default(),
15640 UnixNanos::default(),
15641 );
15642 cache.borrow_mut().add_quote(quote).unwrap();
15643 }
15644
15645 #[tokio::test]
15646 async fn submit_market_order_without_cached_quote_emits_denied() {
15647 let (client, cache, mut rx) = create_execution_client();
15648 let instrument_id = register_test_instrument(&client, &cache);
15649 let mut factory = test_order_factory();
15650 let order = test_market_order(
15651 &mut factory,
15652 instrument_id,
15653 "O-MARKET-NO-QUOTE",
15654 OrderSide::Buy,
15655 );
15656 cache_order(&cache, order.clone());
15657
15658 let command = SubmitOrder::from_order(
15659 &order,
15660 trader_id(),
15661 Some(client_id()),
15662 None,
15663 UUID4::new(),
15664 UnixNanos::default(),
15665 );
15666 let _ = client.submit_order(command);
15668
15669 let event = recv_order_event(&mut rx).await;
15670 match event {
15671 OrderEventAny::Denied(event) => {
15672 assert!(
15673 event.reason.contains("no cached quote"),
15674 "expected no-cached-quote in reason, was {:?}",
15675 event.reason,
15676 );
15677 }
15678 event => panic!("expected denied event, was {event:?}"),
15679 }
15680
15681 assert_nonce_reusable(&client.dispatch);
15682 }
15683
15684 #[tokio::test]
15685 async fn submit_market_buy_with_quote_uses_ask_widened_by_slippage() {
15686 let (client, cache, mut rx) = create_execution_client();
15687 let instrument_id = register_test_instrument(&client, &cache);
15688 add_test_quote(&cache, instrument_id, "2360.00", "2361.00");
15689
15690 let mut factory = test_order_factory();
15691 let order = test_market_order(
15692 &mut factory,
15693 instrument_id,
15694 "O-MARKET-QUOTED-BUY",
15695 OrderSide::Buy,
15696 );
15697 cache_order(&cache, order.clone());
15698
15699 let command = SubmitOrder::from_order(
15700 &order,
15701 trader_id(),
15702 Some(client_id()),
15703 None,
15704 UUID4::new(),
15705 UnixNanos::default(),
15706 );
15707 let _ = client.submit_order(command);
15708
15709 let submitted = recv_order_event(&mut rx).await;
15710 assert!(
15711 matches!(submitted, OrderEventAny::Submitted(_)),
15712 "expected submitted, was {submitted:?}",
15713 );
15714 let rejected = recv_order_event(&mut rx).await;
15715 match rejected {
15716 OrderEventAny::Rejected(event) => {
15717 assert!(
15718 event
15719 .reason
15720 .contains("Lighter submit_order dispatch failed"),
15721 );
15722 }
15723 event => panic!("expected rejected event, was {event:?}"),
15724 }
15725
15726 assert_nonce_reusable(&client.dispatch);
15727 }
15728
15729 #[tokio::test]
15730 async fn submit_order_with_sub_tick_quantity_emits_denied() {
15731 let (client, cache, mut rx) = create_execution_client();
15732 let instrument_id = register_test_instrument(&client, &cache);
15733 let mut factory = test_order_factory();
15734 let order = factory.limit(
15736 instrument_id,
15737 OrderSide::Buy,
15738 Quantity::from("0.00001"),
15739 Price::from("2361.31"),
15740 Some(TimeInForce::Gtc),
15741 None,
15742 Some(false),
15743 Some(false),
15744 None,
15745 None,
15746 None,
15747 None,
15748 None,
15749 None,
15750 None,
15751 Some(ClientOrderId::from("O-SUB-TICK-QTY")),
15752 );
15753 cache_order(&cache, order.clone());
15754
15755 let command = SubmitOrder::from_order(
15756 &order,
15757 trader_id(),
15758 Some(client_id()),
15759 None,
15760 UUID4::new(),
15761 UnixNanos::default(),
15762 );
15763 let _ = client.submit_order(command);
15764
15765 let event = recv_order_event(&mut rx).await;
15766 match event {
15767 OrderEventAny::Denied(event) => {
15768 assert!(
15769 event.reason.contains("rounds to 0 ticks"),
15770 "expected rounds-to-0 in reason, was {:?}",
15771 event.reason,
15772 );
15773 }
15774 event => panic!("expected denied event, was {event:?}"),
15775 }
15776
15777 assert_nonce_reusable(&client.dispatch);
15778 }
15779
15780 #[tokio::test]
15781 async fn submit_order_below_min_notional_emits_denied() {
15782 let (client, cache, mut rx) = create_execution_client();
15783 let instrument_id = register_test_instrument(&client, &cache);
15784 let mut factory = test_order_factory();
15785 let order = factory.limit(
15786 instrument_id,
15787 OrderSide::Buy,
15788 Quantity::from("0.0010"),
15789 Price::from("2361.31"),
15790 Some(TimeInForce::Gtc),
15791 None,
15792 Some(false),
15793 Some(false),
15794 None,
15795 None,
15796 None,
15797 None,
15798 None,
15799 None,
15800 None,
15801 Some(ClientOrderId::from("O-BELOW-MIN-NOTIONAL")),
15802 );
15803 cache_order(&cache, order.clone());
15804
15805 let command = SubmitOrder::from_order(
15806 &order,
15807 trader_id(),
15808 Some(client_id()),
15809 None,
15810 UUID4::new(),
15811 UnixNanos::default(),
15812 );
15813 let _ = client.submit_order(command);
15814
15815 let event = recv_order_event(&mut rx).await;
15816 match event {
15817 OrderEventAny::Denied(event) => {
15818 assert!(
15819 event.reason.contains("min_quote_amount"),
15820 "expected min_quote_amount in reason, was {:?}",
15821 event.reason,
15822 );
15823 }
15824 event => panic!("expected denied event, was {event:?}"),
15825 }
15826
15827 assert_nonce_reusable(&client.dispatch);
15828 }
15829
15830 #[tokio::test]
15831 async fn submit_stop_market_with_sub_tick_trigger_emits_denied() {
15832 let (client, cache, mut rx) = create_execution_client();
15833 let instrument_id = register_test_instrument(&client, &cache);
15834 let mut factory = test_order_factory();
15835 let order = factory.stop_market(
15837 instrument_id,
15838 OrderSide::Buy,
15839 Quantity::from("0.1000"),
15840 Price::from("0.001"),
15841 None,
15842 Some(TimeInForce::Gtc),
15843 None,
15844 Some(false),
15845 Some(false),
15846 None,
15847 None,
15848 None,
15849 None,
15850 None,
15851 None,
15852 Some(ClientOrderId::from("O-STOP-SUB-TICK")),
15853 );
15854 cache_order(&cache, order.clone());
15855
15856 let command = SubmitOrder::from_order(
15857 &order,
15858 trader_id(),
15859 Some(client_id()),
15860 None,
15861 UUID4::new(),
15862 UnixNanos::default(),
15863 );
15864 let _ = client.submit_order(command);
15865
15866 let event = recv_order_event(&mut rx).await;
15867 match event {
15868 OrderEventAny::Denied(event) => {
15869 assert!(
15870 event.reason.contains("rounds to 0 ticks"),
15871 "expected rounds-to-0 in reason, was {:?}",
15872 event.reason,
15873 );
15874 }
15875 event => panic!("expected denied event, was {event:?}"),
15876 }
15877
15878 assert_nonce_reusable(&client.dispatch);
15879 }
15880
15881 #[tokio::test]
15882 async fn submit_stop_market_dispatches_using_trigger_widened_by_slippage() {
15883 let (client, cache, mut rx) = create_execution_client();
15884 let instrument_id = register_test_instrument(&client, &cache);
15885 let mut factory = test_order_factory();
15886 let order = factory.stop_market(
15887 instrument_id,
15888 OrderSide::Sell,
15889 Quantity::from("0.1000"),
15890 Price::from("2300.00"), None,
15892 Some(TimeInForce::Gtc),
15893 None,
15894 Some(false),
15895 Some(false),
15896 None,
15897 None,
15898 None,
15899 None,
15900 None,
15901 None,
15902 Some(ClientOrderId::from("O-STOP-MARKET")),
15903 );
15904 cache_order(&cache, order.clone());
15905
15906 let command = SubmitOrder::from_order(
15907 &order,
15908 trader_id(),
15909 Some(client_id()),
15910 None,
15911 UUID4::new(),
15912 UnixNanos::default(),
15913 );
15914 let _ = client.submit_order(command);
15915
15916 let submitted = recv_order_event(&mut rx).await;
15917 assert!(matches!(submitted, OrderEventAny::Submitted(_)));
15918 let rejected = recv_order_event(&mut rx).await;
15919 assert!(matches!(rejected, OrderEventAny::Rejected(_)));
15920 assert_nonce_reusable(&client.dispatch);
15921 }
15922
15923 #[tokio::test]
15924 async fn submit_market_order_respects_per_order_slippage_override() {
15925 let (client, cache, mut rx) = create_execution_client();
15928 let instrument_id = register_test_instrument(&client, &cache);
15929 add_test_quote(&cache, instrument_id, "2360.00", "2361.00");
15930
15931 let mut factory = test_order_factory();
15932 let order = test_market_order(
15933 &mut factory,
15934 instrument_id,
15935 "O-MARKET-ZERO-SLIP",
15936 OrderSide::Buy,
15937 );
15938 cache_order(&cache, order.clone());
15939
15940 let params: Params =
15941 serde_json::from_value(serde_json::json!({"market_order_slippage_bps": 0})).unwrap();
15942 let mut command = SubmitOrder::from_order(
15943 &order,
15944 trader_id(),
15945 Some(client_id()),
15946 None,
15947 UUID4::new(),
15948 UnixNanos::default(),
15949 );
15950 command.params = Some(params);
15951 let _ = client.submit_order(command);
15952
15953 let submitted = recv_order_event(&mut rx).await;
15954 assert!(matches!(submitted, OrderEventAny::Submitted(_)));
15955 let rejected = recv_order_event(&mut rx).await;
15956 assert!(matches!(rejected, OrderEventAny::Rejected(_)));
15957 assert_nonce_reusable(&client.dispatch);
15958 }
15959
15960 #[tokio::test]
15961 async fn resolve_slippage_bps_prefers_params_over_config_default() {
15962 let (client, _cache, _rx) = create_execution_client();
15963 assert_eq!(client.resolve_slippage_bps(None), 50);
15964
15965 let override_params: Params =
15966 serde_json::from_value(serde_json::json!({"market_order_slippage_bps": 100})).unwrap();
15967 assert_eq!(client.resolve_slippage_bps(Some(&override_params)), 100);
15968
15969 let unrelated_params: Params =
15970 serde_json::from_value(serde_json::json!({"other_key": 999})).unwrap();
15971 assert_eq!(client.resolve_slippage_bps(Some(&unrelated_params)), 50);
15972 }
15973
15974 #[tokio::test]
15975 async fn submit_market_sell_with_quote_uses_bid_widened_by_slippage() {
15976 let (client, cache, mut rx) = create_execution_client();
15977 let instrument_id = register_test_instrument(&client, &cache);
15978 add_test_quote(&cache, instrument_id, "2360.00", "2361.00");
15979
15980 let mut factory = test_order_factory();
15981 let order = test_market_order(
15982 &mut factory,
15983 instrument_id,
15984 "O-MARKET-QUOTED-SELL",
15985 OrderSide::Sell,
15986 );
15987 cache_order(&cache, order.clone());
15988
15989 let command = SubmitOrder::from_order(
15990 &order,
15991 trader_id(),
15992 Some(client_id()),
15993 None,
15994 UUID4::new(),
15995 UnixNanos::default(),
15996 );
15997 let _ = client.submit_order(command);
15998
15999 let submitted = recv_order_event(&mut rx).await;
16000 assert!(
16001 matches!(submitted, OrderEventAny::Submitted(_)),
16002 "expected submitted, was {submitted:?}",
16003 );
16004 let rejected = recv_order_event(&mut rx).await;
16005 match rejected {
16006 OrderEventAny::Rejected(event) => {
16007 assert!(
16008 event
16009 .reason
16010 .contains("Lighter submit_order dispatch failed"),
16011 );
16012 }
16013 event => panic!("expected rejected event, was {event:?}"),
16014 }
16015
16016 assert_nonce_reusable(&client.dispatch);
16017 }
16018
16019 #[rstest]
16020 fn integrator_attributes_tag_mainnet_orders() {
16021 assert_eq!(
16022 integrator_attributes(Some(LIGHTER_NAUTILUS_INTEGRATOR_ACCOUNT_INDEX)),
16023 L2TxAttributes {
16024 integrator_account_index: LIGHTER_NAUTILUS_INTEGRATOR_ACCOUNT_INDEX,
16025 ..Default::default()
16026 },
16027 );
16028 }
16029
16030 #[rstest]
16031 #[case("expire_maker", "account_index", 0, 0)]
16032 #[case("expire_maker", "master_account_index", 0, 1)]
16033 #[case("expire_taker", "account_index", 1, 0)]
16034 #[case("EXPIRE_TAKER", "MASTER_ACCOUNT_INDEX", 1, 1)]
16035 #[case("cancel_both", "account_index", 2, 0)]
16036 #[case("cancel_both", "master_account_index", 2, 1)]
16037 #[case("reduce", "account_index", 3, 0)]
16038 fn resolve_self_trade_params(
16039 #[case] behavior: &str,
16040 #[case] equality: &str,
16041 #[case] expected_behavior: u8,
16042 #[case] expected_equality: u8,
16043 ) {
16044 let (client, _cache, _rx) = create_execution_client();
16045 let params: Params = serde_json::from_value(serde_json::json!({
16046 "self_trade_behavior_mode": behavior,
16047 "self_trade_equality_mode": equality,
16048 }))
16049 .unwrap();
16050
16051 let attrs = client.resolve_create_attributes(Some(¶ms)).unwrap();
16052
16053 assert_eq!(
16054 attrs,
16055 L2TxAttributes {
16056 self_trade_behavior_mode: Some(
16057 serde_json::from_value(serde_json::json!(expected_behavior)).unwrap()
16058 ),
16059 self_trade_equality_mode: Some(
16060 serde_json::from_value(serde_json::json!(expected_equality)).unwrap()
16061 ),
16062 ..Default::default()
16063 }
16064 );
16065 }
16066
16067 #[rstest]
16068 fn resolve_self_trade_defaults_are_explicit() {
16069 let (client, _cache, _rx) = create_execution_client();
16070 let attrs = client.resolve_create_attributes(None).unwrap();
16071
16072 assert_eq!(
16073 attrs,
16074 L2TxAttributes {
16075 self_trade_behavior_mode: Some(LighterSelfTradeBehaviorMode::ExpireMaker),
16076 self_trade_equality_mode: Some(LighterSelfTradeEqualityMode::AccountIndex),
16077 ..Default::default()
16078 }
16079 );
16080 }
16081
16082 #[rstest]
16083 #[case(serde_json::json!({"self_trade_behavior_mode": "unknown"}), "invalid Lighter self_trade_behavior_mode")]
16084 #[case(serde_json::json!({"self_trade_equality_mode": "unknown"}), "invalid Lighter self_trade_equality_mode")]
16085 #[case(serde_json::json!({"self_trade_behavior_mode": 1}), "Lighter self_trade_behavior_mode must be a string")]
16086 #[case(serde_json::json!({"self_trade_equality_mode": null}), "Lighter self_trade_equality_mode must be a string")]
16087 #[case(serde_json::json!({"self_trade_behavior_mode": "reduce", "self_trade_equality_mode": "master_account_index"}), "Lighter reduce self-trade behavior cannot combine with master-account equality")]
16088 fn resolve_self_trade_rejects_invalid_params(
16089 #[case] value: serde_json::Value,
16090 #[case] expected: &str,
16091 ) {
16092 let (client, _cache, _rx) = create_execution_client();
16093 let params: Params = serde_json::from_value(value).unwrap();
16094
16095 let error = client.resolve_create_attributes(Some(¶ms)).unwrap_err();
16096
16097 assert_eq!(error.to_string(), expected);
16098 assert_nonce_reusable(&client.dispatch);
16099 }
16100
16101 #[rstest]
16102 #[case::single(false, false)]
16103 #[case::list(true, false)]
16104 #[case::group(true, true)]
16105 #[tokio::test]
16106 async fn submit_self_trade_invalid_combination_denies_before_dispatch(
16107 #[case] list: bool,
16108 #[case] grouped: bool,
16109 ) {
16110 let (client, cache, mut rx) = create_execution_client();
16111 let instrument = register_test_instrument(&client, &cache);
16112 let mut factory = test_order_factory();
16113
16114 let orders = if grouped {
16115 test_native_group(
16116 instrument,
16117 LighterGroupingType::OneTriggersOneCancelsTheOther,
16118 )
16119 } else {
16120 vec![test_limit_order(
16121 &mut factory,
16122 instrument,
16123 "O-SELF-TRADE-INVALID",
16124 )]
16125 };
16126
16127 for order in &orders {
16128 cache_order(&cache, order.clone());
16129 }
16130
16131 let params: Params = serde_json::from_value(serde_json::json!({
16132 "self_trade_behavior_mode": "reduce",
16133 "self_trade_equality_mode": "master_account_index",
16134 }))
16135 .unwrap();
16136
16137 if list {
16138 let mut command = submit_order_list_command(&orders, "OL-SELF-TRADE-INVALID");
16139 command.params = Some(params);
16140 client.submit_order_list(command).unwrap();
16141 } else {
16142 let mut command = SubmitOrder::from_order(
16143 &orders[0],
16144 trader_id(),
16145 Some(client_id()),
16146 None,
16147 UUID4::new(),
16148 UnixNanos::default(),
16149 );
16150 command.params = Some(params);
16151 client.submit_order(command).unwrap();
16152 }
16153
16154 for order in &orders {
16155 match recv_order_event(&mut rx).await {
16156 OrderEventAny::Denied(event) => {
16157 assert_eq!(event.client_order_id, order.client_order_id());
16158 assert!(event.reason.contains("Lighter reduce self-trade behavior cannot combine with master-account equality"));
16159 }
16160 event => panic!("expected OrderDenied, was {event:?}"),
16161 }
16162 }
16163
16164 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
16165 assert_nonce_reusable(&client.dispatch);
16166 }
16167
16168 #[rstest]
16169 fn integrator_attributes_leave_testnet_orders_unattributed() {
16170 assert_eq!(integrator_attributes(None), L2TxAttributes::default());
16171 }
16172
16173 #[rstest]
16174 #[case::lighter_mainnet(
16175 LighterDeployment::Lighter,
16176 LighterEnvironment::Mainnet,
16177 Some(LIGHTER_NAUTILUS_INTEGRATOR_ACCOUNT_INDEX)
16178 )]
16179 #[case::lighter_testnet(LighterDeployment::Lighter, LighterEnvironment::Testnet, None)]
16180 #[case::robinhood_mainnet(
16181 LighterDeployment::Robinhood,
16182 LighterEnvironment::Mainnet,
16183 Some(LIGHTER_ROBINHOOD_NAUTILUS_INTEGRATOR_ACCOUNT_INDEX)
16184 )]
16185 #[case::robinhood_testnet(LighterDeployment::Robinhood, LighterEnvironment::Testnet, None)]
16186 fn integrator_account_index_follows_deployment(
16187 #[case] deployment: LighterDeployment,
16188 #[case] environment: LighterEnvironment,
16189 #[case] expected: Option<u64>,
16190 ) {
16191 let mut config = test_config();
16192 config.deployment = deployment;
16193 config.environment = environment;
16194 config.venue = Some(*LIGHTER_VENUE);
16195 let (client, _cache, _rx) = create_execution_client_with_config(config);
16196
16197 assert_eq!(client.integrator_account_index(), expected);
16198 }
16199
16200 #[rstest]
16201 fn robinhood_orders_carry_integrator_attributes() {
16202 let mut config = test_config();
16203 config.deployment = LighterDeployment::Robinhood;
16204 config.environment = LighterEnvironment::Mainnet;
16205
16206 assert_eq!(
16207 integrator_attributes(deployment::integrator_account_index(
16208 config.deployment,
16209 config.environment,
16210 )),
16211 L2TxAttributes {
16212 integrator_account_index: LIGHTER_ROBINHOOD_NAUTILUS_INTEGRATOR_ACCOUNT_INDEX,
16213 ..Default::default()
16214 },
16215 );
16216 }
16217
16218 use std::str::FromStr;
16219
16220 use nautilus_live::ExecutionEventEmitter;
16221 use rust_decimal::Decimal;
16222
16223 use crate::{
16224 common::enums::{
16225 LighterOrderKind, LighterOrderSide, LighterOrderStatus, LighterOrderTimeInForce,
16226 LighterTradeType, LighterTriggerStatus,
16227 },
16228 http::models::{LighterOrder, LighterTrade},
16229 };
16230
16231 fn dispatcher_emitter() -> (
16232 ExecutionEventEmitter,
16233 tokio::sync::mpsc::UnboundedReceiver<ExecutionEvent>,
16234 ) {
16235 let mut emitter = ExecutionEventEmitter::new(
16236 get_atomic_clock_realtime(),
16237 trader_id(),
16238 account_id(),
16239 AccountType::Margin,
16240 None,
16241 );
16242 let (sender, receiver) = tokio::sync::mpsc::unbounded_channel::<ExecutionEvent>();
16243 emitter.set_sender(sender);
16244 (emitter, receiver)
16245 }
16246
16247 struct DispatcherRig {
16251 dispatch: WsDispatchState,
16252 registry: Arc<MarketRegistry>,
16253 emitter: ExecutionEventEmitter,
16254 rx: tokio::sync::mpsc::UnboundedReceiver<ExecutionEvent>,
16255 instrument_id: InstrumentId,
16256 cloid: ClientOrderId,
16257 }
16258
16259 fn dispatcher_rig(cloid_suffix: &str) -> DispatcherRig {
16260 let registry = Arc::new(MarketRegistry::new());
16261 let instrument_id = registry.insert(TEST_MARKET_INDEX, "ETH", LighterProductType::Perp);
16266 let instrument = InstrumentAny::CryptoPerpetual(
16267 CryptoPerpetual::builder()
16268 .instrument_id(instrument_id)
16269 .raw_symbol(Symbol::new("ETH-PERP"))
16270 .base_currency(Currency::from("ETH"))
16271 .quote_currency(Currency::from("USDC"))
16272 .settlement_currency(Currency::from("USDC"))
16273 .is_inverse(false)
16274 .price_precision(2)
16275 .size_precision(4)
16276 .price_increment(Price::from("0.01"))
16277 .size_increment(Quantity::from("0.0001"))
16278 .min_notional(Money::from("10.000000 USDC"))
16279 .ts_event(UnixNanos::default())
16280 .ts_init(UnixNanos::default())
16281 .build()
16282 .unwrap(),
16283 );
16284 LIGHTER_INSTRUMENT_CACHE.insert(instrument_id, instrument);
16285 let (emitter, rx) = dispatcher_emitter();
16286
16287 DispatcherRig {
16288 dispatch: WsDispatchState::new(),
16289 registry,
16290 emitter,
16291 rx,
16292 instrument_id,
16293 cloid: ClientOrderId::new(format!("CLOID-{cloid_suffix}")),
16294 }
16295 }
16296
16297 fn register_identity(rig: &DispatcherRig) {
16298 register_identity_for(rig, OrderType::Limit);
16299 }
16300
16301 fn register_identity_for(rig: &DispatcherRig, order_type: OrderType) {
16302 let derived = rig.dispatch.derive_client_order_index(&rig.cloid);
16303 let client_order_index = rig.dispatch.register_cloid(derived, rig.cloid).unwrap();
16304 rig.dispatch.register_order_identity(
16305 rig.cloid,
16306 OrderIdentity::new(
16307 rig.instrument_id,
16308 strategy_id(),
16309 OrderSide::Buy,
16310 order_type,
16311 client_order_index,
16312 ),
16313 );
16314 rig.dispatch
16315 .mark_order_submission(&rig.cloid, TEST_SUBMISSION_NONCE);
16316 }
16317
16318 fn register_unowned_identity(rig: &DispatcherRig) {
16319 let derived = rig.dispatch.derive_client_order_index(&rig.cloid);
16320 let client_order_index = rig.dispatch.register_cloid(derived, rig.cloid).unwrap();
16321 rig.dispatch.register_order_identity(
16322 rig.cloid,
16323 OrderIdentity::new(
16324 rig.instrument_id,
16325 strategy_id(),
16326 OrderSide::Buy,
16327 OrderType::Limit,
16328 client_order_index,
16329 ),
16330 );
16331 }
16332
16333 fn dispatcher_test_order(rig: &DispatcherRig, status: LighterOrderStatus) -> LighterOrder {
16334 let derived = rig.dispatch.derive_client_order_index(&rig.cloid);
16335 rig.dispatch.register_cloid(derived, rig.cloid).unwrap();
16336
16337 LighterOrder {
16338 order_index: 281_476_929_510_110,
16339 client_order_index: derived,
16340 order_id: "281476929510110".to_string(),
16341 client_order_id: derived.to_string(),
16342 market_index: TEST_MARKET_INDEX,
16343 owner_account_index: TEST_ACCOUNT_INDEX_I64,
16344 initial_base_amount: Decimal::from_str("0.0050").unwrap(),
16345 price: Decimal::from_str("2352.74").unwrap(),
16346 nonce: TEST_ORDER_NONCE,
16347 remaining_base_amount: Decimal::from_str("0.0050").unwrap(),
16348 is_ask: false,
16349 base_size: 50,
16350 base_price: 235_274,
16351 filled_base_amount: Decimal::ZERO,
16352 filled_quote_amount: Decimal::ZERO,
16353 side: Some(LighterOrderSide::Buy),
16354 order_type: LighterOrderKind::Limit,
16355 time_in_force: LighterOrderTimeInForce::GoodTillTime,
16356 reduce_only: false,
16357 trigger_price: Decimal::ZERO,
16358 order_expiry: 1_780_360_584_479,
16359 status,
16360 trigger_status: LighterTriggerStatus::Na,
16361 trigger_time: 0,
16362 parent_order_index: 0,
16363 parent_order_id: "0".to_string(),
16364 to_trigger_order_id_0: "0".to_string(),
16365 to_trigger_order_id_1: "0".to_string(),
16366 to_cancel_order_id_0: "0".to_string(),
16367 integrator_fee_collector_index: "0".to_string(),
16368 integrator_taker_fee: Decimal::ZERO,
16369 integrator_maker_fee: Decimal::ZERO,
16370 block_height: 227_535_532,
16371 timestamp: 1_777_941_383_576,
16372 created_at: 1_777_941_383_576,
16373 updated_at: 1_777_941_383_900,
16374 transaction_time: 1_777_941_383_576_735,
16375 }
16376 }
16377
16378 fn dispatcher_test_trade(rig: &DispatcherRig, user_is_bidder: bool) -> LighterTrade {
16379 let derived = rig.dispatch.derive_client_order_index(&rig.cloid);
16380 rig.dispatch.register_cloid(derived, rig.cloid).unwrap();
16381
16382 LighterTrade {
16383 trade_id: 19_209_006_902,
16384 trade_id_str: Some("19209006902".to_string()),
16385 tx_hash: "000000128b1ee814".to_string(),
16386 trade_type: LighterTradeType::Trade,
16387 market_id: TEST_MARKET_INDEX,
16388 size: Decimal::from_str("0.1336").unwrap(),
16389 price: Decimal::from_str("2352.73").unwrap(),
16390 usd_amount: Decimal::from_str("314.324728").unwrap(),
16391 ask_id: 281_476_929_510_102,
16392 ask_id_str: Some("281476929510102".to_string()),
16393 bid_id: 562_947_905_631_053,
16394 bid_id_str: Some("562947905631053".to_string()),
16395 ask_client_id: if user_is_bidder { 0 } else { derived },
16396 ask_client_id_str: Some(if user_is_bidder {
16397 "0".to_string()
16398 } else {
16399 derived.to_string()
16400 }),
16401 bid_client_id: if user_is_bidder { derived } else { 0 },
16402 bid_client_id_str: Some(if user_is_bidder {
16403 derived.to_string()
16404 } else {
16405 "0".to_string()
16406 }),
16407 ask_account_id: if user_is_bidder {
16408 91_249
16409 } else {
16410 TEST_ACCOUNT_INDEX_I64
16411 },
16412 bid_account_id: if user_is_bidder {
16413 TEST_ACCOUNT_INDEX_I64
16414 } else {
16415 91_249
16416 },
16417 is_maker_ask: false,
16418 block_height: 227_535_535,
16419 timestamp: 1_777_941_384_181,
16420 taker_fee: Some(196),
16421 taker_position_size_before: None,
16422 taker_entry_quote_before: None,
16423 taker_initial_margin_fraction_before: None,
16424 taker_position_sign_changed: None,
16425 maker_fee: Some(28),
16426 maker_position_size_before: None,
16427 maker_entry_quote_before: None,
16428 maker_initial_margin_fraction_before: None,
16429 maker_position_sign_changed: None,
16430 transaction_time: 1_777_941_384_181_586,
16431 ask_account_pnl: None,
16432 bid_account_pnl: None,
16433 }
16434 }
16435
16436 fn drain_events(
16440 rx: &mut tokio::sync::mpsc::UnboundedReceiver<ExecutionEvent>,
16441 ) -> Vec<ExecutionEvent> {
16442 let mut events = Vec::new();
16443 while let Ok(event) = rx.try_recv() {
16444 events.push(event);
16445 }
16446
16447 events
16448 }
16449
16450 #[rstest]
16451 fn dispatch_lighter_order_tracked_emits_accepted_then_silent_repeat() {
16452 let mut rig = dispatcher_rig("1");
16453 register_identity(&rig);
16454 let order = dispatcher_test_order(&rig, LighterOrderStatus::Open);
16455
16456 dispatch_lighter_order(
16457 &order,
16458 &rig.dispatch,
16459 &rig.emitter,
16460 &rig.registry,
16461 account_id(),
16462 trader_id(),
16463 UnixNanos::from(1),
16464 );
16465 dispatch_lighter_order(
16466 &order,
16467 &rig.dispatch,
16468 &rig.emitter,
16469 &rig.registry,
16470 account_id(),
16471 trader_id(),
16472 UnixNanos::from(2),
16473 );
16474
16475 let events = drain_events(&mut rig.rx);
16476 assert_eq!(
16477 events.len(),
16478 1,
16479 "exactly one event expected, was {events:?}",
16480 );
16481
16482 match &events[0] {
16483 ExecutionEvent::Order(OrderEventAny::Accepted(e)) => {
16484 assert_eq!(e.client_order_id, rig.cloid);
16485 assert_eq!(e.venue_order_id.to_string(), "281476929510110");
16486 }
16487 other => panic!("expected Accepted, was {other:?}"),
16488 }
16489
16490 assert!(rig.dispatch.accepted_was_emitted(&rig.cloid));
16491 assert!(rig.dispatch.snapshot_for(&rig.cloid).is_some());
16492 }
16493
16494 #[rstest]
16495 fn dispatch_lighter_order_tracked_emits_updated_on_shape_change() {
16496 let mut rig = dispatcher_rig("2");
16497 register_identity(&rig);
16498 let order = dispatcher_test_order(&rig, LighterOrderStatus::Open);
16499
16500 dispatch_lighter_order(
16501 &order,
16502 &rig.dispatch,
16503 &rig.emitter,
16504 &rig.registry,
16505 account_id(),
16506 trader_id(),
16507 UnixNanos::from(1),
16508 );
16509 assert_eq!(drain_events(&mut rig.rx).len(), 1);
16510
16511 let mut modified = order;
16512 modified.price = Decimal::from_str("2400.00").unwrap();
16513
16514 dispatch_lighter_order(
16515 &modified,
16516 &rig.dispatch,
16517 &rig.emitter,
16518 &rig.registry,
16519 account_id(),
16520 trader_id(),
16521 UnixNanos::from(2),
16522 );
16523
16524 let events = drain_events(&mut rig.rx);
16525 assert_eq!(
16526 events.len(),
16527 1,
16528 "expected one Updated event, was {events:?}",
16529 );
16530
16531 match &events[0] {
16532 ExecutionEvent::Order(OrderEventAny::Updated(e)) => {
16533 assert_eq!(e.client_order_id, rig.cloid);
16534 assert_eq!(e.price, Some(Price::from("2400.00")));
16535 }
16536 other => panic!("expected Updated, was {other:?}"),
16537 }
16538
16539 let snapshot = rig.dispatch.snapshot_for(&rig.cloid).expect("snapshot");
16540 assert_eq!(snapshot.price, Some(Price::from("2400.00")));
16541 }
16542
16543 #[rstest]
16544 #[case(LighterOrderStatus::Pending)]
16545 #[case(LighterOrderStatus::Open)]
16546 fn dispatch_lighter_order_defers_live_frame_while_cancel_pending(
16547 #[case] status: LighterOrderStatus,
16548 ) {
16549 let mut rig = dispatcher_rig(match status {
16550 LighterOrderStatus::Pending => "24-PENDING",
16551 LighterOrderStatus::Open => "24-OPEN",
16552 other => panic!("unexpected case {other:?}"),
16553 });
16554
16555 register_identity(&rig);
16556 let order = dispatcher_test_order(&rig, LighterOrderStatus::Open);
16557
16558 dispatch_lighter_order(
16559 &order,
16560 &rig.dispatch,
16561 &rig.emitter,
16562 &rig.registry,
16563 account_id(),
16564 trader_id(),
16565 UnixNanos::from(1),
16566 );
16567 assert_eq!(drain_events(&mut rig.rx).len(), 1);
16568
16569 rig.dispatch
16570 .set_pending_order_action(rig.cloid, PendingOrderAction::Cancel);
16571 let mut late = order;
16572 late.status = status;
16573 late.initial_base_amount = Decimal::from_str("0.0075").unwrap();
16574 late.price = Decimal::from_str("2400.00").unwrap();
16575
16576 dispatch_lighter_order(
16577 &late,
16578 &rig.dispatch,
16579 &rig.emitter,
16580 &rig.registry,
16581 account_id(),
16582 trader_id(),
16583 UnixNanos::from(2),
16584 );
16585
16586 let events = drain_events(&mut rig.rx);
16587 assert_eq!(events.len(), 0, "late frame must be silent: {events:?}");
16588 assert_eq!(
16589 rig.dispatch.pending_order_action(&rig.cloid),
16590 Some(PendingOrderAction::Cancel),
16591 );
16592 assert!(rig.dispatch.accepted_was_emitted(&rig.cloid));
16593 assert!(!rig.dispatch.triggered_was_emitted(&rig.cloid));
16594 assert_eq!(
16595 rig.dispatch.lookup_venue_order_id(&rig.cloid),
16596 Some(VenueOrderId::new("281476929510110")),
16597 );
16598 assert!(rig.dispatch.order_identities.contains_key(&rig.cloid));
16599 assert_eq!(
16600 rig.dispatch.snapshot_for(&rig.cloid),
16601 Some(crate::websocket::dispatch::OrderShapeSnapshot {
16602 quantity: Quantity::from("0.0050"),
16603 price: Some(Price::from("2352.74")),
16604 trigger_price: None,
16605 }),
16606 );
16607 }
16608
16609 #[rstest]
16610 #[case::standalone(false)]
16611 #[case::native_child(true)]
16612 fn dispatch_lighter_order_emits_updated_then_triggered_for_one_frame(
16613 #[case] native_child: bool,
16614 ) {
16615 let mut rig = dispatcher_rig("25");
16616
16617 if native_child {
16618 let child = OrderTestBuilder::new(OrderType::StopLimit)
16619 .trader_id(trader_id())
16620 .strategy_id(strategy_id())
16621 .instrument_id(rig.instrument_id)
16622 .client_order_id(rig.cloid)
16623 .side(OrderSide::Buy)
16624 .quantity(Quantity::from("0.0050"))
16625 .price(Price::from("2352.74"))
16626 .trigger_price(Price::from("2300.00"))
16627 .parent_order_id(ClientOrderId::new("TRIGGER-PARENT"))
16628 .build();
16629 rig.dispatch.register_create_identity(&child).unwrap();
16630 rig.dispatch
16631 .mark_order_submission(&rig.cloid, TEST_SUBMISSION_NONCE);
16632 } else {
16633 register_identity_for(&rig, OrderType::StopLimit);
16634 }
16635
16636 let mut order = dispatcher_test_order(&rig, LighterOrderStatus::Pending);
16637 order.order_type = LighterOrderKind::StopLossLimit;
16638 order.trigger_price = Decimal::from_str("2300.00").unwrap();
16639
16640 dispatch_lighter_order(
16641 &order,
16642 &rig.dispatch,
16643 &rig.emitter,
16644 &rig.registry,
16645 account_id(),
16646 trader_id(),
16647 UnixNanos::from(1),
16648 );
16649 let accepted = drain_events(&mut rig.rx);
16650 assert_eq!(accepted.len(), 1, "expected one acceptance: {accepted:?}");
16651 assert!(matches!(
16652 accepted[0],
16653 ExecutionEvent::Order(OrderEventAny::Accepted(_)),
16654 ));
16655
16656 let command = ModifyOrder::new(
16657 trader_id(),
16658 Some(client_id()),
16659 strategy_id(),
16660 rig.instrument_id,
16661 rig.cloid,
16662 Some(VenueOrderId::new("281476929510110")),
16663 Some(Quantity::from("0.0075")),
16664 Some(Price::from("2400.00")),
16665 Some(Price::from("2390.00")),
16666 UUID4::new(),
16667 UnixNanos::from(2),
16668 None,
16669 None,
16670 );
16671 rig.dispatch.set_pending_modify_action(&command);
16672 rig.dispatch.set_pending_modify_shape(
16673 &command,
16674 OrderShapeSnapshot {
16675 quantity: command.quantity.unwrap(),
16676 price: command.price,
16677 trigger_price: command.trigger_price,
16678 },
16679 );
16680
16681 rig.dispatch
16682 .set_pending_modify_nonce(&rig.cloid, command.command_id, 0, 29);
16683 let mut triggered = order;
16684 triggered.status = LighterOrderStatus::Open;
16685 triggered.trigger_status = LighterTriggerStatus::Ready;
16686 triggered.initial_base_amount = Decimal::from_str("0.0075").unwrap();
16687 triggered.remaining_base_amount = triggered.initial_base_amount;
16688 triggered.price = Decimal::from_str("2400.00").unwrap();
16689 triggered.trigger_price = Decimal::from_str("2390.00").unwrap();
16690
16691 dispatch_lighter_order(
16692 &triggered,
16693 &rig.dispatch,
16694 &rig.emitter,
16695 &rig.registry,
16696 account_id(),
16697 trader_id(),
16698 UnixNanos::from(2),
16699 );
16700
16701 assert!(drain_events(&mut rig.rx).is_empty());
16702 let (client, _, _) = create_execution_client();
16703 let mut context = acked_probe_context(&client);
16704 context.dispatch = rig.dispatch.clone();
16705 context.registry = rig.registry.clone();
16706 context.emitter = rig.emitter.clone();
16707
16708 let pending = PendingSendTx {
16709 batch_id: None,
16710 connection_epoch: 0,
16711 kind: PendingSendTxKind::Modify {
16712 strategy_id: strategy_id(),
16713 instrument_id: rig.instrument_id,
16714 client_order_id: rig.cloid,
16715 venue_order_id: command.venue_order_id,
16716 },
16717 submitted_at: UnixNanos::from(2),
16718 nonce: 29,
16719 api_key_index: TEST_API_KEY_INDEX,
16720 tx_hash: "confirmed-trigger".to_string(),
16721 };
16722
16723 confirm_modify_order(&pending, &context, &triggered).unwrap();
16724 let events = drain_events(&mut rig.rx);
16725 assert_eq!(
16726 events.len(),
16727 2,
16728 "expected Updated then Triggered: {events:?}",
16729 );
16730
16731 match &events[0] {
16732 ExecutionEvent::Order(OrderEventAny::Updated(event)) => {
16733 assert_eq!(event.trader_id, trader_id());
16734 assert_eq!(event.strategy_id, strategy_id());
16735 assert_eq!(event.instrument_id, rig.instrument_id);
16736 assert_eq!(event.client_order_id, rig.cloid);
16737 assert_eq!(
16738 event.venue_order_id,
16739 Some(VenueOrderId::new("281476929510110")),
16740 );
16741 assert_eq!(event.account_id, Some(account_id()));
16742 assert_eq!(event.quantity, Quantity::from("0.0075"));
16743 assert_eq!(event.price, Some(Price::from("2400.00")));
16744 assert_eq!(event.trigger_price, Some(Price::from("2390.00")));
16745 }
16746 other => panic!("first event should be Updated, was {other:?}"),
16747 }
16748
16749 match &events[1] {
16750 ExecutionEvent::Order(OrderEventAny::Triggered(event)) => {
16751 assert_eq!(event.trader_id, trader_id());
16752 assert_eq!(event.strategy_id, strategy_id());
16753 assert_eq!(event.instrument_id, rig.instrument_id);
16754 assert_eq!(event.client_order_id, rig.cloid);
16755 assert_eq!(
16756 event.venue_order_id,
16757 Some(VenueOrderId::new("281476929510110")),
16758 );
16759 assert_eq!(event.account_id, Some(account_id()));
16760 }
16761 other => panic!("second event should be Triggered, was {other:?}"),
16762 }
16763
16764 assert_eq!(rig.dispatch.pending_order_action(&rig.cloid), None);
16765 assert!(rig.dispatch.accepted_was_emitted(&rig.cloid));
16766 assert!(rig.dispatch.triggered_was_emitted(&rig.cloid));
16767 assert_eq!(
16768 rig.dispatch.snapshot_for(&rig.cloid),
16769 Some(crate::websocket::dispatch::OrderShapeSnapshot {
16770 quantity: Quantity::from("0.0075"),
16771 price: Some(Price::from("2400.00")),
16772 trigger_price: Some(Price::from("2390.00")),
16773 }),
16774 );
16775
16776 dispatch_lighter_order(
16777 &triggered,
16778 &rig.dispatch,
16779 &rig.emitter,
16780 &rig.registry,
16781 account_id(),
16782 trader_id(),
16783 UnixNanos::from(3),
16784 );
16785 assert!(
16786 drain_events(&mut rig.rx).is_empty(),
16787 "replayed frame must be silent",
16788 );
16789 }
16790
16791 #[rstest]
16792 #[case(LighterTriggerStatus::Ready)]
16793 #[case(LighterTriggerStatus::Na)]
16794 fn dispatch_lighter_order_defers_trigger_until_modify_resolves(
16795 #[case] trigger_status: LighterTriggerStatus,
16796 ) {
16797 let mut rig = dispatcher_rig("26");
16798 register_identity_for(&rig, OrderType::StopLimit);
16799 let mut order = dispatcher_test_order(&rig, LighterOrderStatus::Pending);
16800 order.order_type = LighterOrderKind::StopLossLimit;
16801 order.trigger_price = Decimal::from_str("2300.00").unwrap();
16802
16803 dispatch_lighter_order(
16804 &order,
16805 &rig.dispatch,
16806 &rig.emitter,
16807 &rig.registry,
16808 account_id(),
16809 trader_id(),
16810 UnixNanos::from(1),
16811 );
16812 assert_eq!(drain_events(&mut rig.rx).len(), 1);
16813
16814 let command = ModifyOrder::new(
16815 trader_id(),
16816 Some(client_id()),
16817 strategy_id(),
16818 rig.instrument_id,
16819 rig.cloid,
16820 Some(VenueOrderId::new("281476929510110")),
16821 Some(Quantity::from("0.0075")),
16822 Some(Price::from("2400.00")),
16823 Some(Price::from("2390.00")),
16824 UUID4::new(),
16825 UnixNanos::from(2),
16826 None,
16827 None,
16828 );
16829 rig.dispatch.set_pending_modify_action(&command);
16830 rig.dispatch.set_pending_modify_shape(
16831 &command,
16832 OrderShapeSnapshot {
16833 quantity: command.quantity.unwrap(),
16834 price: command.price,
16835 trigger_price: command.trigger_price,
16836 },
16837 );
16838
16839 order.status = LighterOrderStatus::Open;
16840 order.trigger_status = trigger_status;
16841 dispatch_lighter_order(
16842 &order,
16843 &rig.dispatch,
16844 &rig.emitter,
16845 &rig.registry,
16846 account_id(),
16847 trader_id(),
16848 UnixNanos::from(2),
16849 );
16850
16851 assert!(drain_events(&mut rig.rx).is_empty());
16852 assert_eq!(
16853 rig.dispatch.pending_order_action(&rig.cloid),
16854 Some(PendingOrderAction::Modify),
16855 );
16856 assert!(!rig.dispatch.triggered_was_emitted(&rig.cloid));
16857
16858 assert!(
16859 rig.dispatch
16860 .clear_pending_modify_if_command(&rig.cloid, command.command_id)
16861 );
16862 dispatch_lighter_order(
16863 &order,
16864 &rig.dispatch,
16865 &rig.emitter,
16866 &rig.registry,
16867 account_id(),
16868 trader_id(),
16869 UnixNanos::from(3),
16870 );
16871
16872 let events = drain_events(&mut rig.rx);
16873 assert_eq!(events.len(), 1, "expected deferred trigger: {events:?}");
16874
16875 match &events[0] {
16876 ExecutionEvent::Order(OrderEventAny::Triggered(event)) => {
16877 assert_eq!(event.client_order_id, rig.cloid);
16878 assert_eq!(
16879 event.venue_order_id,
16880 Some(VenueOrderId::new("281476929510110")),
16881 );
16882 assert_eq!(event.account_id, Some(account_id()));
16883 }
16884 other => panic!("expected Triggered, was {other:?}"),
16885 }
16886
16887 assert!(rig.dispatch.triggered_was_emitted(&rig.cloid));
16888 }
16889
16890 #[rstest]
16891 fn dispatch_lighter_order_pending_emits_accepted_then_updated() {
16892 let mut rig = dispatcher_rig("23");
16893 register_identity(&rig);
16894 let order = dispatcher_test_order(&rig, LighterOrderStatus::Pending);
16895
16896 dispatch_lighter_order(
16897 &order,
16898 &rig.dispatch,
16899 &rig.emitter,
16900 &rig.registry,
16901 account_id(),
16902 trader_id(),
16903 UnixNanos::from(1),
16904 );
16905 let accepted = drain_events(&mut rig.rx);
16906
16907 let mut modified = order;
16908 modified.price = Decimal::from_str("2400.00").unwrap();
16909 dispatch_lighter_order(
16910 &modified,
16911 &rig.dispatch,
16912 &rig.emitter,
16913 &rig.registry,
16914 account_id(),
16915 trader_id(),
16916 UnixNanos::from(2),
16917 );
16918 let updated = drain_events(&mut rig.rx);
16919
16920 assert_eq!(
16921 accepted.len(),
16922 1,
16923 "expected one Accepted event: {accepted:?}"
16924 );
16925 assert!(matches!(
16926 accepted[0],
16927 ExecutionEvent::Order(OrderEventAny::Accepted(_))
16928 ));
16929 assert_eq!(updated.len(), 1, "expected one Updated event: {updated:?}");
16930 assert!(matches!(
16931 updated[0],
16932 ExecutionEvent::Order(OrderEventAny::Updated(_))
16933 ));
16934 assert_eq!(
16935 rig.dispatch
16936 .snapshot_for(&rig.cloid)
16937 .expect("snapshot")
16938 .price,
16939 Some(Price::from("2400.00")),
16940 );
16941 }
16942
16943 #[rstest]
16944 fn dispatch_lighter_order_untracked_emits_report() {
16945 let mut rig = dispatcher_rig("3");
16946 let mut order = dispatcher_test_order(&rig, LighterOrderStatus::Open);
16948 order.client_order_id = "external-1".to_string();
16949 order.client_order_index = 0;
16950
16951 dispatch_lighter_order(
16952 &order,
16953 &rig.dispatch,
16954 &rig.emitter,
16955 &rig.registry,
16956 account_id(),
16957 trader_id(),
16958 UnixNanos::from(1),
16959 );
16960
16961 let events = drain_events(&mut rig.rx);
16962 assert_eq!(events.len(), 1);
16963
16964 match &events[0] {
16965 ExecutionEvent::Report(report) => match report {
16966 EngineExecutionReport::Order(r) => {
16967 assert_eq!(r.venue_order_id.to_string(), "281476929510110");
16968 }
16969 other => panic!("expected order report, was {other:?}"),
16970 },
16971 other => panic!("expected report, was {other:?}"),
16972 }
16973
16974 assert!(
16975 !rig.dispatch
16976 .accepted_was_emitted(&ClientOrderId::new("external-1"))
16977 );
16978 }
16979
16980 #[rstest]
16981 fn same_index_external_order_waits_for_create_submission_ownership() {
16982 let mut rig = dispatcher_rig("EPOCH-ORDER");
16983 register_unowned_identity(&rig);
16984 let order = dispatcher_test_order(&rig, LighterOrderStatus::Open);
16985 let venue_order_id = VenueOrderId::new(order.order_id.as_str());
16986
16987 dispatch_lighter_order(
16988 &order,
16989 &rig.dispatch,
16990 &rig.emitter,
16991 &rig.registry,
16992 account_id(),
16993 trader_id(),
16994 UnixNanos::from(1),
16995 );
16996
16997 let external = drain_events(&mut rig.rx);
16998 assert_eq!(external.len(), 1);
16999
17000 match &external[0] {
17001 ExecutionEvent::Report(EngineExecutionReport::Order(report)) => {
17002 assert_eq!(report.venue_order_id, venue_order_id);
17003 assert_eq!(
17004 report.client_order_id,
17005 Some(ClientOrderId::new(venue_order_id.as_str())),
17006 );
17007 }
17008 other => panic!("expected one external order report, was {other:?}"),
17009 }
17010
17011 assert!(!rig.dispatch.accepted_was_emitted(&rig.cloid));
17012 assert!(!rig.dispatch.venue_id_map.contains_key(&rig.cloid));
17013
17014 rig.dispatch
17015 .mark_order_submission(&rig.cloid, TEST_SUBMISSION_NONCE);
17016 dispatch_lighter_order(
17017 &order,
17018 &rig.dispatch,
17019 &rig.emitter,
17020 &rig.registry,
17021 account_id(),
17022 trader_id(),
17023 UnixNanos::from(2),
17024 );
17025
17026 let replayed_external = drain_events(&mut rig.rx);
17027 assert_eq!(replayed_external.len(), 1);
17028
17029 match &replayed_external[0] {
17030 ExecutionEvent::Report(EngineExecutionReport::Order(report)) => {
17031 assert_eq!(report.venue_order_id, venue_order_id);
17032 assert_eq!(
17033 report.client_order_id,
17034 Some(ClientOrderId::new(venue_order_id.as_str())),
17035 );
17036 }
17037 other => panic!("expected replayed external order report, was {other:?}"),
17038 }
17039
17040 assert!(!rig.dispatch.accepted_was_emitted(&rig.cloid));
17041 assert!(!rig.dispatch.venue_id_map.contains_key(&rig.cloid));
17042
17043 let mut local_order = order.clone();
17044 local_order.order_id = "281476929510111".to_string();
17045 let local_venue_order_id = VenueOrderId::new(local_order.order_id.as_str());
17046 dispatch_lighter_order(
17047 &local_order,
17048 &rig.dispatch,
17049 &rig.emitter,
17050 &rig.registry,
17051 account_id(),
17052 trader_id(),
17053 UnixNanos::from(3),
17054 );
17055
17056 let local = drain_events(&mut rig.rx);
17057 assert_eq!(local.len(), 1);
17058
17059 match &local[0] {
17060 ExecutionEvent::Order(OrderEventAny::Accepted(event)) => {
17061 assert_eq!(event.client_order_id, rig.cloid);
17062 assert_eq!(event.venue_order_id, local_venue_order_id);
17063 assert_eq!(event.account_id, account_id());
17064 }
17065 other => panic!("expected one local OrderAccepted, was {other:?}"),
17066 }
17067
17068 assert_eq!(
17069 rig.dispatch
17070 .venue_id_map
17071 .get(&rig.cloid)
17072 .map(|entry| *entry),
17073 Some(local_venue_order_id),
17074 );
17075 assert_eq!(
17076 rig.dispatch
17077 .resolve_live_order_cloid(order.client_order_id.as_str(), local_venue_order_id,),
17078 Some(rig.cloid),
17079 "a bound identity remains stable across replay epochs",
17080 );
17081 }
17082
17083 #[rstest]
17084 #[case::order_first(false)]
17085 #[case::trade_first(true)]
17086 fn quantity_amend_immediate_fill_orders_update_before_fill(
17087 #[case] trade_first: bool,
17088 #[values(false, true)] reduced: bool,
17089 ) {
17090 let mut rig = dispatcher_rig("AMEND-FILL");
17091 let mut cached = OrderTestBuilder::new(OrderType::Limit)
17092 .trader_id(trader_id())
17093 .strategy_id(strategy_id())
17094 .instrument_id(rig.instrument_id)
17095 .client_order_id(rig.cloid)
17096 .side(OrderSide::Buy)
17097 .quantity(Quantity::from("0.0050"))
17098 .price(Price::from("2352.74"))
17099 .build();
17100 rig.dispatch.register_create_identity(&cached).unwrap();
17101 rig.dispatch
17102 .mark_order_submission(&rig.cloid, TEST_SUBMISSION_NONCE);
17103 let mut frame = dispatcher_test_order(&rig, LighterOrderStatus::Open);
17104 dispatch_lighter_order(
17105 &frame,
17106 &rig.dispatch,
17107 &rig.emitter,
17108 &rig.registry,
17109 account_id(),
17110 trader_id(),
17111 UnixNanos::from(1),
17112 );
17113
17114 for event in drain_events(&mut rig.rx) {
17115 let ExecutionEvent::Order(event) = event else {
17116 panic!("expected order event");
17117 };
17118
17119 cached.apply(event).unwrap();
17120 }
17121
17122 let command = ModifyOrder::new(
17123 trader_id(),
17124 Some(client_id()),
17125 strategy_id(),
17126 rig.instrument_id,
17127 rig.cloid,
17128 cached.venue_order_id(),
17129 Some(Quantity::from("0.0100")),
17130 Some(Price::from("2400.00")),
17131 None,
17132 UUID4::new(),
17133 UnixNanos::from(2),
17134 None,
17135 None,
17136 );
17137 rig.dispatch.set_pending_modify_action(&command);
17138 rig.dispatch.set_pending_modify_shape(
17139 &command,
17140 OrderShapeSnapshot {
17141 quantity: Quantity::from("0.0100"),
17142 price: Some(Price::from("2400.00")),
17143 trigger_price: None,
17144 },
17145 );
17146
17147 rig.dispatch
17148 .set_pending_modify_nonce(&rig.cloid, command.command_id, 0, 19);
17149
17150 let pending = PendingSendTx {
17151 batch_id: None,
17152 connection_epoch: 0,
17153 kind: PendingSendTxKind::Modify {
17154 strategy_id: strategy_id(),
17155 instrument_id: rig.instrument_id,
17156 client_order_id: rig.cloid,
17157 venue_order_id: cached.venue_order_id(),
17158 },
17159 submitted_at: UnixNanos::from(2),
17160 nonce: 19,
17161 api_key_index: TEST_API_KEY_INDEX,
17162 tx_hash: "confirmed-amend".to_string(),
17163 };
17164
17165 let mut trade = dispatcher_test_trade(&rig, true);
17166 trade.bid_id = frame.order_index;
17167 trade.bid_id_str = Some(frame.order_id.clone());
17168 let expected_qty = Quantity::from(if reduced { "0.0040" } else { "0.0100" });
17169 trade.size = expected_qty.as_decimal();
17170 trade.usd_amount = trade.size * trade.price;
17171
17172 if trade_first {
17173 dispatch_lighter_trade(
17174 &trade,
17175 &rig.dispatch,
17176 &rig.emitter,
17177 &rig.registry,
17178 account_id(),
17179 trader_id(),
17180 Some(TEST_ACCOUNT_INDEX_I64),
17181 UnixNanos::from(3),
17182 );
17183 assert!(
17184 drain_events(&mut rig.rx).is_empty(),
17185 "unconfirmed amend fill must wait"
17186 );
17187 assert!(
17188 !rig.dispatch
17189 .seen_trade_ids
17190 .contains(&parse_lighter_trade_id(&trade).unwrap())
17191 );
17192 }
17193
17194 frame.status = LighterOrderStatus::Filled;
17195 frame.initial_base_amount = Decimal::from_str("0.0100").unwrap();
17196 frame.remaining_base_amount = Decimal::ZERO;
17197 frame.filled_base_amount = trade.size;
17198 frame.price = Decimal::from_str("2400.00").unwrap();
17199 dispatch_lighter_order(
17200 &frame,
17201 &rig.dispatch,
17202 &rig.emitter,
17203 &rig.registry,
17204 account_id(),
17205 trader_id(),
17206 UnixNanos::from(4),
17207 );
17208
17209 if !trade_first {
17210 dispatch_lighter_trade(
17211 &trade,
17212 &rig.dispatch,
17213 &rig.emitter,
17214 &rig.registry,
17215 account_id(),
17216 trader_id(),
17217 Some(TEST_ACCOUNT_INDEX_I64),
17218 UnixNanos::from(5),
17219 );
17220 }
17221
17222 assert!(
17223 drain_events(&mut rig.rx).is_empty(),
17224 "order echo must wait for transaction confirmation"
17225 );
17226 let (client, _, _) = create_execution_client();
17227 let mut context = acked_probe_context(&client);
17228 context.dispatch = rig.dispatch.clone();
17229 context.registry = rig.registry.clone();
17230 context.emitter = rig.emitter.clone();
17231 confirm_modify_order(&pending, &context, &frame).unwrap();
17232 let events = drain_events(&mut rig.rx);
17233 assert_eq!(events.len(), 2, "expected update then fill, was {events:?}");
17234
17235 let ExecutionEvent::Order(OrderEventAny::Updated(update)) = &events[0] else {
17236 panic!("expected update before fill, was {:?}", events[0]);
17237 };
17238
17239 assert_eq!(update.client_order_id, rig.cloid);
17240 assert_eq!(update.quantity, expected_qty);
17241 assert_eq!(update.price, Some(Price::from("2400.00")));
17242 assert_eq!(update.trigger_price, None);
17243
17244 let ExecutionEvent::Order(OrderEventAny::Filled(fill)) = &events[1] else {
17245 panic!("expected real fill, was {:?}", events[1]);
17246 };
17247
17248 assert_eq!(fill.trade_id, parse_lighter_trade_id(&trade).unwrap());
17249 assert_eq!(fill.last_qty, expected_qty);
17250 assert_eq!(fill.last_px, Price::from("2352.73"));
17251
17252 for event in events {
17253 let ExecutionEvent::Order(event) = event else {
17254 panic!("expected order event");
17255 };
17256
17257 cached.apply(event).unwrap();
17258 }
17259
17260 assert_eq!(cached.quantity(), expected_qty);
17261 assert_eq!(cached.filled_qty(), expected_qty);
17262 assert_eq!(cached.price(), Some(Price::from("2400.00")));
17263 assert_eq!(cached.status(), OrderStatus::Filled);
17264 dispatch_lighter_order(
17265 &frame,
17266 &rig.dispatch,
17267 &rig.emitter,
17268 &rig.registry,
17269 account_id(),
17270 trader_id(),
17271 UnixNanos::from(6),
17272 );
17273 dispatch_lighter_trade(
17274 &trade,
17275 &rig.dispatch,
17276 &rig.emitter,
17277 &rig.registry,
17278 account_id(),
17279 trader_id(),
17280 Some(TEST_ACCOUNT_INDEX_I64),
17281 UnixNanos::from(7),
17282 );
17283 assert!(
17284 drain_events(&mut rig.rx)
17285 .iter()
17286 .all(|event| matches!(event, ExecutionEvent::Report(_))),
17287 "terminal replay must not emit duplicate order events"
17288 );
17289 }
17290
17291 #[rstest]
17292 #[case(OrderType::StopMarket)]
17293 #[case(OrderType::StopLimit)]
17294 #[case(OrderType::MarketIfTouched)]
17295 #[case(OrderType::LimitIfTouched)]
17296 fn conditional_fill_synthesizes_trigger_once_across_retirement(#[case] kind: OrderType) {
17297 let mut rig = dispatcher_rig("CONDITIONAL-FILL");
17298 register_identity_for(&rig, kind);
17299 let trade = dispatcher_test_trade(&rig, true);
17300 dispatch_lighter_trade(
17301 &trade,
17302 &rig.dispatch,
17303 &rig.emitter,
17304 &rig.registry,
17305 account_id(),
17306 trader_id(),
17307 Some(TEST_ACCOUNT_INDEX_I64),
17308 UnixNanos::from(1),
17309 );
17310 let first = drain_events(&mut rig.rx);
17311 let triggerable = TRIGGERABLE_ORDER_TYPES.contains(&kind);
17312 let mut builder = OrderTestBuilder::new(kind);
17313 builder
17314 .trader_id(trader_id())
17315 .strategy_id(strategy_id())
17316 .instrument_id(rig.instrument_id)
17317 .client_order_id(rig.cloid)
17318 .side(OrderSide::Buy)
17319 .quantity(Quantity::from("0.2672"))
17320 .trigger_price(Price::from("2300.00"));
17321
17322 if LIMIT_ORDER_TYPES.contains(&kind) {
17323 builder.price(Price::from("2352.74"));
17324 }
17325
17326 let mut order = builder.build();
17327
17328 for event in &first {
17329 let ExecutionEvent::Order(event) = event else {
17330 panic!("expected order event");
17331 };
17332
17333 order.apply(event.clone()).unwrap();
17334 }
17335
17336 rig.dispatch.retire_order_identity(&rig.cloid);
17337 let mut subsequent = trade;
17338 subsequent.trade_id += 1;
17339 subsequent.trade_id_str = Some(subsequent.trade_id.to_string());
17340 dispatch_lighter_trade(
17341 &subsequent,
17342 &rig.dispatch,
17343 &rig.emitter,
17344 &rig.registry,
17345 account_id(),
17346 trader_id(),
17347 Some(TEST_ACCOUNT_INDEX_I64),
17348 UnixNanos::from(2),
17349 );
17350 let next = drain_events(&mut rig.rx);
17351 assert_eq!(first.len(), 2 + usize::from(triggerable));
17352 assert!(
17353 matches!(&first[0], ExecutionEvent::Order(OrderEventAny::Accepted(event)) if event.client_order_id == rig.cloid)
17354 );
17355
17356 if triggerable {
17357 assert!(
17358 matches!(&first[1], ExecutionEvent::Order(OrderEventAny::Triggered(event)) if event.client_order_id == rig.cloid)
17359 );
17360 }
17361
17362 assert!(
17363 matches!(first.last().unwrap(), ExecutionEvent::Order(OrderEventAny::Filled(event)) if event.client_order_id == rig.cloid && event.order_type == kind && event.last_qty == Quantity::from("0.1336"))
17364 );
17365 assert_eq!(next.len(), 1);
17366 assert!(
17367 matches!(&next[0], ExecutionEvent::Order(OrderEventAny::Filled(event)) if event.client_order_id == rig.cloid && event.trade_id.as_str() == subsequent.trade_id.to_string())
17368 );
17369
17370 let ExecutionEvent::Order(event) = &next[0] else {
17371 panic!("expected trailing fill");
17372 };
17373
17374 order.apply(event.clone()).unwrap();
17375 assert_eq!(order.status(), OrderStatus::Filled);
17376 assert_eq!(order.filled_qty(), Quantity::from("0.2672"));
17377 assert_eq!(order.is_triggered(), Some(triggerable));
17378 assert_eq!(rig.dispatch.triggered_was_emitted(&rig.cloid), triggerable);
17379 }
17380
17381 #[rstest]
17382 #[case(OrderType::Limit)]
17383 #[case(OrderType::StopLimit)]
17384 #[case(OrderType::StopMarket)]
17385 fn cached_order_restores_before_first_fill(
17386 #[case] kind: OrderType,
17387 #[values(false, true)] retired: bool,
17388 ) {
17389 let mut rig = dispatcher_rig("CACHED-FIRST-FILL");
17390 let trade = dispatcher_test_trade(&rig, true);
17391 rig.dispatch.forget_cloid(trade.bid_client_id);
17392 let venue = VenueOrderId::new(trade.bid_id_str.as_deref().unwrap());
17393 let mut builder = OrderTestBuilder::new(kind);
17394 builder
17395 .trader_id(trader_id())
17396 .strategy_id(strategy_id())
17397 .instrument_id(rig.instrument_id)
17398 .client_order_id(rig.cloid)
17399 .side(OrderSide::Buy)
17400 .quantity(Quantity::from("0.1336"));
17401
17402 if LIMIT_ORDER_TYPES.contains(&kind) {
17403 builder.price(Price::from("2352.74"));
17404 }
17405
17406 if STOP_ORDER_TYPES.contains(&kind) {
17407 builder.trigger_price(Price::from("2300.00"));
17408 }
17409
17410 let mut order = builder.build();
17411 order
17412 .apply(OrderEventAny::Accepted(OrderAccepted::new(
17413 trader_id(),
17414 strategy_id(),
17415 rig.instrument_id,
17416 rig.cloid,
17417 venue,
17418 account_id(),
17419 UUID4::new(),
17420 UnixNanos::from(1),
17421 UnixNanos::from(1),
17422 false,
17423 )))
17424 .unwrap();
17425 rig.dispatch.seed_order_restorations(vec![order.clone()]);
17426 if retired {
17427 rig.dispatch
17428 .restore_reconciled_order(&order, trade.bid_client_id, venue, true)
17429 .unwrap();
17430 }
17431
17432 dispatch_lighter_trade(
17433 &trade,
17434 &rig.dispatch,
17435 &rig.emitter,
17436 &rig.registry,
17437 account_id(),
17438 trader_id(),
17439 Some(TEST_ACCOUNT_INDEX_I64),
17440 UnixNanos::from(2),
17441 );
17442 let events = drain_events(&mut rig.rx);
17443 let triggerable = TRIGGERABLE_ORDER_TYPES.contains(&kind);
17444 assert_eq!(events.len(), 1 + usize::from(triggerable));
17445
17446 for event in events {
17447 let ExecutionEvent::Order(event) = event else {
17448 panic!("cached fill must remain tracked");
17449 };
17450
17451 assert_eq!(event.client_order_id(), rig.cloid);
17452 order.apply(event).unwrap();
17453 }
17454
17455 assert_eq!(order.status(), OrderStatus::Filled);
17456 assert_eq!(order.filled_qty(), Quantity::from("0.1336"));
17457 assert_eq!(order.venue_order_id(), Some(venue));
17458 assert_eq!(
17459 rig.dispatch.client_order_index(&rig.cloid),
17460 Some(trade.bid_client_id)
17461 );
17462 }
17463
17464 #[rstest]
17465 fn dispatch_lighter_trade_tracked_synthesizes_accepted_before_filled() {
17466 let mut rig = dispatcher_rig("4");
17470 register_identity(&rig);
17471
17472 let trade = dispatcher_test_trade(&rig, true);
17473
17474 dispatch_lighter_trade(
17475 &trade,
17476 &rig.dispatch,
17477 &rig.emitter,
17478 &rig.registry,
17479 account_id(),
17480 trader_id(),
17481 Some(TEST_ACCOUNT_INDEX_I64),
17482 UnixNanos::from(1),
17483 );
17484
17485 let events = drain_events(&mut rig.rx);
17486 assert_eq!(
17487 events.len(),
17488 2,
17489 "expected Accepted then Filled, was {events:?}",
17490 );
17491
17492 match &events[0] {
17493 ExecutionEvent::Order(OrderEventAny::Accepted(_)) => {}
17494 other => panic!("first event should be Accepted, was {other:?}"),
17495 }
17496
17497 match &events[1] {
17498 ExecutionEvent::Order(OrderEventAny::Filled(e)) => {
17499 assert_eq!(e.client_order_id, rig.cloid);
17500 assert_eq!(e.last_qty, Quantity::from("0.1336"));
17501 assert_eq!(e.last_px, Price::from("2352.73"));
17502 }
17503 other => panic!("second event should be Filled, was {other:?}"),
17504 }
17505
17506 assert!(rig.dispatch.accepted_was_emitted(&rig.cloid));
17507 }
17508
17509 #[rstest]
17510 fn dispatch_lighter_trade_dedupes_repeated_trade_ids() {
17511 let mut rig = dispatcher_rig("5");
17512 register_identity(&rig);
17513 let trade = dispatcher_test_trade(&rig, true);
17514
17515 for _ in 0..3 {
17516 dispatch_lighter_trade(
17517 &trade,
17518 &rig.dispatch,
17519 &rig.emitter,
17520 &rig.registry,
17521 account_id(),
17522 trader_id(),
17523 Some(TEST_ACCOUNT_INDEX_I64),
17524 UnixNanos::from(1),
17525 );
17526 }
17527
17528 let events = drain_events(&mut rig.rx);
17529 assert_eq!(
17531 events.len(),
17532 2,
17533 "expected dedup after first dispatch, was {events:?}"
17534 );
17535 }
17536
17537 #[rstest]
17538 #[case::commission_overflow(true)]
17539 #[case::invalid_timestamp(false)]
17540 fn invalid_tracked_fill_preserves_unaccepted_order_for_replay(#[case] overflow: bool) {
17541 let mut rig = dispatcher_rig("INVALID-FILL-REPLAY");
17542 register_identity(&rig);
17543 let trade = dispatcher_test_trade(&rig, true);
17544 let mut invalid = trade.clone();
17545 if overflow {
17546 invalid.price = Decimal::from(1_000_000_000);
17547 invalid.size = Decimal::from(1_000_000_000);
17548 } else {
17549 invalid.timestamp = -1;
17550 }
17551
17552 let trade_id = parse_lighter_trade_id(&trade).unwrap();
17553
17554 if overflow {
17555 let error = parse_lighter_order_filled(
17556 &invalid,
17557 &LIGHTER_INSTRUMENT_CACHE.get(&rig.instrument_id).unwrap(),
17558 &rig.dispatch.order_identity(&rig.cloid).unwrap(),
17559 rig.cloid,
17560 account_id(),
17561 trader_id(),
17562 TEST_ACCOUNT_INDEX_I64,
17563 UnixNanos::from(1),
17564 )
17565 .unwrap_err();
17566 assert!(is_commission_error(&error));
17567 }
17568
17569 dispatch_lighter_trade(
17570 &invalid,
17571 &rig.dispatch,
17572 &rig.emitter,
17573 &rig.registry,
17574 account_id(),
17575 trader_id(),
17576 Some(TEST_ACCOUNT_INDEX_I64),
17577 UnixNanos::from(1),
17578 );
17579
17580 assert!(drain_events(&mut rig.rx).is_empty());
17581 assert!(!rig.dispatch.accepted_was_emitted(&rig.cloid));
17582 assert!(!rig.dispatch.seen_trade_ids.contains(&trade_id));
17583 assert!(
17584 !rig.dispatch
17585 .order_identity(&rig.cloid)
17586 .unwrap()
17587 .has_publication()
17588 );
17589 assert!(rig.dispatch.order_identities.contains_key(&rig.cloid));
17590
17591 dispatch_lighter_trade(
17592 &trade,
17593 &rig.dispatch,
17594 &rig.emitter,
17595 &rig.registry,
17596 account_id(),
17597 trader_id(),
17598 Some(TEST_ACCOUNT_INDEX_I64),
17599 UnixNanos::from(2),
17600 );
17601
17602 let events = drain_events(&mut rig.rx);
17603 assert_eq!(events.len(), 2);
17604 assert!(
17605 matches!(&events[0], ExecutionEvent::Order(OrderEventAny::Accepted(event)) if event.client_order_id == rig.cloid)
17606 );
17607
17608 let ExecutionEvent::Order(OrderEventAny::Filled(actual)) = &events[1] else {
17609 panic!("expected typed fill");
17610 };
17611
17612 assert_eq!(actual.trade_id, trade_id);
17613 assert_eq!(actual.trader_id, trader_id());
17614 assert_eq!(actual.strategy_id, strategy_id());
17615 assert_eq!(actual.instrument_id, rig.instrument_id);
17616 assert_eq!(actual.client_order_id, rig.cloid);
17617 assert_eq!(actual.venue_order_id, VenueOrderId::from("562947905631053"));
17618 assert_eq!(actual.account_id, account_id());
17619 assert_eq!(actual.order_side, OrderSide::Buy);
17620 assert_eq!(actual.order_type, OrderType::Limit);
17621 assert_eq!(actual.last_qty, Quantity::from("0.1336"));
17622 assert_eq!(actual.last_px, Price::from("2352.73"));
17623 assert_eq!(actual.currency, Currency::USDC());
17624 assert_eq!(actual.liquidity_side, LiquiditySide::Maker);
17625 assert_eq!(actual.ts_event, UnixNanos::from(1_777_941_384_181_000_000));
17626 assert_eq!(actual.ts_init, UnixNanos::from(2));
17627 assert!(!actual.reconciliation);
17628 assert_eq!(actual.position_id, None);
17629 assert_eq!(actual.commission, Some(Money::from("0.00880109 USDC")));
17630 assert_eq!(actual.info, None);
17631 assert_eq!(actual.causation_id, None);
17632 assert!(rig.dispatch.seen_trade_ids.contains(&trade_id));
17633 }
17634
17635 #[rstest]
17636 #[case::tracked(true)]
17637 #[case::external(false)]
17638 fn malformed_duplicate_preserves_delivered_fill(#[case] tracked: bool) {
17639 let mut rig = dispatcher_rig("DUPLICATE-MALFORMED");
17640 if tracked {
17641 register_identity(&rig);
17642 }
17643
17644 let original = dispatcher_test_trade(&rig, true);
17645 let mut malformed = original.clone();
17646 malformed.timestamp = -1;
17647 for trade in [&original, &malformed, &original] {
17648 dispatch_lighter_trade(
17649 trade,
17650 &rig.dispatch,
17651 &rig.emitter,
17652 &rig.registry,
17653 account_id(),
17654 trader_id(),
17655 Some(TEST_ACCOUNT_INDEX_I64),
17656 UnixNanos::from(1),
17657 );
17658 }
17659
17660 let events = drain_events(&mut rig.rx);
17661 assert_eq!(events.len(), if tracked { 2 } else { 1 });
17662
17663 let fills = events
17664 .iter()
17665 .filter(|event| {
17666 matches!(
17667 event,
17668 ExecutionEvent::Order(OrderEventAny::Filled(_)) | ExecutionEvent::Report(_)
17669 )
17670 })
17671 .count();
17672
17673 assert_eq!(fills, 1);
17674 assert!(
17675 rig.dispatch
17676 .seen_trade_ids
17677 .contains(&parse_lighter_trade_id(&original).unwrap())
17678 );
17679 }
17680
17681 #[rstest]
17682 #[case::confirmed(true, false)]
17683 #[case::confirmed_stale_http(true, true)]
17684 #[case::rejected(false, false)]
17685 fn cancel_overlap_preserves_amendment_and_real_fill(
17686 #[case] confirmed: bool,
17687 #[case] stale_http: bool,
17688 ) {
17689 let mut rig = dispatcher_rig("AMEND-CANCEL-FILL");
17690 let mut cached = OrderTestBuilder::new(OrderType::Limit)
17691 .trader_id(trader_id())
17692 .strategy_id(strategy_id())
17693 .instrument_id(rig.instrument_id)
17694 .client_order_id(rig.cloid)
17695 .side(OrderSide::Buy)
17696 .quantity(Quantity::from("0.0050"))
17697 .price(Price::from("2352.74"))
17698 .build();
17699 rig.dispatch.register_create_identity(&cached).unwrap();
17700 rig.dispatch
17701 .mark_order_submission(&rig.cloid, TEST_SUBMISSION_NONCE);
17702 let mut raw = dispatcher_test_order(&rig, LighterOrderStatus::Open);
17703 dispatch_lighter_order(
17704 &raw,
17705 &rig.dispatch,
17706 &rig.emitter,
17707 &rig.registry,
17708 account_id(),
17709 trader_id(),
17710 UnixNanos::from(1),
17711 );
17712
17713 for event in drain_events(&mut rig.rx) {
17714 let ExecutionEvent::Order(event) = event else {
17715 panic!("expected acceptance");
17716 };
17717
17718 cached.apply(event).unwrap();
17719 }
17720
17721 let command = ModifyOrder::new(
17722 trader_id(),
17723 Some(client_id()),
17724 strategy_id(),
17725 rig.instrument_id,
17726 rig.cloid,
17727 cached.venue_order_id(),
17728 Some(Quantity::from("0.0100")),
17729 Some(Price::from("2400.00")),
17730 None,
17731 UUID4::new(),
17732 UnixNanos::from(2),
17733 None,
17734 None,
17735 );
17736 assert!(rig.dispatch.set_pending_modify_action(&command));
17737 rig.dispatch.set_pending_modify_shape(
17738 &command,
17739 OrderShapeSnapshot {
17740 quantity: command.quantity.unwrap(),
17741 price: command.price,
17742 trigger_price: None,
17743 },
17744 );
17745
17746 rig.dispatch
17747 .set_pending_modify_nonce(&rig.cloid, command.command_id, 0, 19);
17748 rig.dispatch
17749 .set_pending_order_action(rig.cloid, PendingOrderAction::Cancel);
17750 rig.dispatch
17751 .set_pending_cancel_nonce(&rig.cloid, 0, 20, true);
17752 assert_eq!(rig.dispatch.pending_modify_nonce(&rig.cloid), Some((0, 19)));
17753 assert_eq!(rig.dispatch.pending_cancel_nonce(&rig.cloid), Some((0, 20)));
17754 let mut trade = dispatcher_test_trade(&rig, true);
17755 trade.bid_id = raw.order_index;
17756 trade.bid_id_str = Some(raw.order_id.clone());
17757 trade.size = Decimal::from_str("0.0030").unwrap();
17758 trade.usd_amount = trade.size * trade.price;
17759 dispatch_lighter_trade(
17760 &trade,
17761 &rig.dispatch,
17762 &rig.emitter,
17763 &rig.registry,
17764 account_id(),
17765 trader_id(),
17766 Some(TEST_ACCOUNT_INDEX_I64),
17767 UnixNanos::from(3),
17768 );
17769 raw.status = LighterOrderStatus::Canceled;
17770 raw.filled_base_amount = trade.size;
17771 if confirmed {
17772 raw.initial_base_amount = Decimal::from_str("0.0100").unwrap();
17773 raw.price = Decimal::from_str("2400.00").unwrap();
17774 }
17775
17776 raw.remaining_base_amount = raw.initial_base_amount - trade.size;
17777 dispatch_lighter_order(
17778 &raw,
17779 &rig.dispatch,
17780 &rig.emitter,
17781 &rig.registry,
17782 account_id(),
17783 trader_id(),
17784 UnixNanos::from(4),
17785 );
17786 assert!(drain_events(&mut rig.rx).is_empty());
17787 assert!(
17788 !rig.dispatch
17789 .seen_trade_ids
17790 .contains(&parse_lighter_trade_id(&trade).unwrap())
17791 );
17792
17793 let pending = PendingSendTx {
17794 batch_id: None,
17795 connection_epoch: 0,
17796 kind: PendingSendTxKind::Modify {
17797 strategy_id: strategy_id(),
17798 instrument_id: rig.instrument_id,
17799 client_order_id: rig.cloid,
17800 venue_order_id: cached.venue_order_id(),
17801 },
17802 submitted_at: UnixNanos::from(2),
17803 nonce: 19,
17804 api_key_index: TEST_API_KEY_INDEX,
17805 tx_hash: "amend-cancel".to_string(),
17806 };
17807
17808 if confirmed {
17809 let (client, _, _) = create_execution_client();
17810 let mut context = acked_probe_context(&client);
17811 context.dispatch = rig.dispatch.clone();
17812 context.registry = rig.registry.clone();
17813 context.emitter = rig.emitter.clone();
17814 let mut http = raw;
17815 if stale_http {
17816 http.status = LighterOrderStatus::Open;
17817 }
17818
17819 confirm_modify_order(&pending, &context, &http).unwrap();
17820 } else {
17821 reject_modify_order(
17822 &rig.dispatch,
17823 &rig.emitter,
17824 &pending,
17825 "amendment failed",
17826 UnixNanos::from(5),
17827 );
17828 }
17829
17830 let events = drain_events(&mut rig.rx);
17831 assert_eq!(events.len(), 3);
17832
17833 match &events[0] {
17834 ExecutionEvent::Order(OrderEventAny::Updated(update)) if confirmed => {
17835 assert_eq!(update.client_order_id, rig.cloid);
17836 assert_eq!(update.quantity, Quantity::from("0.0100"));
17837 assert_eq!(update.price, Some(Price::from("2400.00")));
17838 }
17839 ExecutionEvent::Order(OrderEventAny::ModifyRejected(rejected)) if !confirmed => {
17840 assert_eq!(rejected.client_order_id, rig.cloid);
17841 assert_eq!(rejected.reason.as_str(), "amendment failed");
17842 }
17843 other => panic!("expected amendment resolution before fill, was {other:?}"),
17844 }
17845
17846 let ExecutionEvent::Order(OrderEventAny::Filled(fill)) = &events[1] else {
17847 panic!("expected held real fill");
17848 };
17849
17850 assert_eq!(fill.client_order_id, rig.cloid);
17851 assert_eq!(fill.trade_id, parse_lighter_trade_id(&trade).unwrap());
17852 assert_eq!(fill.last_qty, Quantity::from("0.0030"));
17853 assert_eq!(fill.last_px, Price::from("2352.73"));
17854
17855 let ExecutionEvent::Order(OrderEventAny::Canceled(cancel)) = &events[2] else {
17856 panic!("expected cancellation after fill");
17857 };
17858
17859 assert_eq!(cancel.client_order_id, rig.cloid);
17860 assert_eq!(cancel.venue_order_id, cached.venue_order_id());
17861
17862 for event in events {
17863 let ExecutionEvent::Order(event) = event else {
17864 panic!("expected order event");
17865 };
17866
17867 cached.apply(event).unwrap();
17868 }
17869
17870 assert_eq!(cached.status(), OrderStatus::Canceled);
17871 assert_eq!(
17872 cached.quantity(),
17873 Quantity::from(if confirmed { "0.0100" } else { "0.0050" })
17874 );
17875 assert_eq!(cached.filled_qty(), Quantity::from("0.0030"));
17876 assert_eq!(
17877 cached.leaves_qty(),
17878 Quantity::from(if confirmed { "0.0070" } else { "0.0020" })
17879 );
17880 assert_eq!(rig.dispatch.pending_modify_nonce(&rig.cloid), None);
17881 assert_eq!(rig.dispatch.pending_cancel_nonce(&rig.cloid), None);
17882 assert!(
17883 rig.dispatch
17884 .seen_trade_ids
17885 .contains(&parse_lighter_trade_id(&trade).unwrap())
17886 );
17887 }
17888
17889 #[rstest]
17890 fn overlapping_modify_rejection_preserves_first_submission() {
17891 let (client, cache, mut rx) = create_execution_client();
17892 let instrument_id = register_test_instrument(&client, &cache);
17893 let cloid = ClientOrderId::from("OVERLAPPING-MODIFY");
17894 let venue = VenueOrderId::from("789");
17895 enqueue_modify(&client, instrument_id, cloid, venue, 23);
17896
17897 let command = ModifyOrder::new(
17898 trader_id(),
17899 Some(client_id()),
17900 strategy_id(),
17901 instrument_id,
17902 cloid,
17903 Some(venue),
17904 Some(Quantity::from("0.3000")),
17905 Some(Price::from("2363.00")),
17906 None,
17907 UUID4::new(),
17908 UnixNanos::from(2),
17909 None,
17910 None,
17911 );
17912 assert!(!client.admit_modify_order(&command));
17913 let events = drain_events(&mut rx);
17914 assert_eq!(events.len(), 1);
17915
17916 let ExecutionEvent::Order(OrderEventAny::ModifyRejected(event)) = &events[0] else {
17917 panic!("expected overlap rejection");
17918 };
17919
17920 assert_eq!(event.trader_id, trader_id());
17921 assert_eq!(event.strategy_id, strategy_id());
17922 assert_eq!(event.instrument_id, instrument_id);
17923 assert_eq!(event.client_order_id, cloid);
17924 assert_eq!(event.venue_order_id, Some(venue));
17925 assert_eq!(event.account_id, Some(account_id()));
17926 assert_eq!(
17927 event.reason.as_str(),
17928 "Lighter permits one unresolved modification per order; wait for its update or rejection"
17929 );
17930 assert!(!event.reconciliation);
17931 assert_eq!(client.dispatch.pending_modify_nonce(&cloid), Some((0, 23)));
17932 assert_eq!(
17933 client.dispatch.pending_modify_shape(&cloid),
17934 Some(OrderShapeSnapshot {
17935 quantity: Quantity::from("0.2000"),
17936 price: Some(Price::from("2362.00")),
17937 trigger_price: None
17938 })
17939 );
17940 assert_eq!(client.dispatch.pending_sendtx_len(), 1);
17941 }
17942
17943 #[rstest]
17944 fn recovery_waits_for_confirmed_update_in_cache() {
17945 let (client, cache, mut rx) = create_execution_client();
17946 let instrument_id = register_test_instrument(&client, &cache);
17947 let rig = dispatcher_rig("HTTP-AMEND");
17948 let cloid = rig.cloid;
17949 let mut raw = dispatcher_test_order(&rig, LighterOrderStatus::Filled);
17950 let venue = VenueOrderId::new(&raw.order_id);
17951 let order = OrderTestBuilder::new(OrderType::Limit)
17952 .trader_id(trader_id())
17953 .strategy_id(strategy_id())
17954 .instrument_id(instrument_id)
17955 .client_order_id(cloid)
17956 .side(OrderSide::Buy)
17957 .quantity(Quantity::from("0.0050"))
17958 .price(Price::from("2352.74"))
17959 .build();
17960 cache_accepted_order(&cache, order, venue, Some(client_id()));
17961 let cached = cache.borrow().order_owned(&cloid).unwrap();
17962 client
17963 .dispatch
17964 .restore_reconciled_order(&cached, raw.client_order_index, venue, false)
17965 .unwrap();
17966 enqueue_modify(&client, instrument_id, cloid, venue, 23);
17967 let pending = client.dispatch.pop_pending_sendtx_head().unwrap();
17968 raw.initial_base_amount = Decimal::from_str("0.2000").unwrap();
17969 raw.filled_base_amount = raw.initial_base_amount;
17970 raw.remaining_base_amount = Decimal::ZERO;
17971 raw.price = Decimal::from_str("2362.00").unwrap();
17972 let instrument = LIGHTER_INSTRUMENT_CACHE.get(&instrument_id).unwrap();
17973 let report =
17974 parse_ws_order_status_report(&raw, &instrument, account_id(), UnixNanos::from(3))
17975 .unwrap()
17976 .with_client_order_id(cloid);
17977 drop(instrument);
17978
17979 let command = GenerateOrderStatusReports::new(
17980 UUID4::new(),
17981 UnixNanos::default(),
17982 false,
17983 Some(instrument_id),
17984 None,
17985 None,
17986 None,
17987 None,
17988 );
17989
17990 let result = finish_order_reports(
17991 &client.core,
17992 &client.dispatch,
17993 OrderReportSweep {
17994 orders: vec![(raw.client_order_index, report)],
17995 ..Default::default()
17996 },
17997 &command,
17998 None,
17999 );
18000
18001 assert_eq!(
18002 result.unwrap_err().to_string(),
18003 "incomplete Lighter order reports: Lighter execution confirmation is pending"
18004 );
18005 assert_eq!(client.dispatch.pending_modify_nonce(&cloid), Some((0, 23)));
18006 assert!(client.dispatch.order_identities.contains_key(&cloid));
18007 let fill = reconciliation_fill_report(
18008 instrument_id,
18009 raw.client_order_index,
18010 venue,
18011 "HTTP-AMEND-TRADE",
18012 );
18013
18014 let withheld = finish_fill_reports(
18015 &client.core,
18016 &client.dispatch,
18017 BTreeSet::new(),
18018 Ok(FillSweep {
18019 reports: vec![fill.clone()],
18020 covers_window: true,
18021 withheld_instruments: AHashSet::new(),
18022 }),
18023 )
18024 .unwrap();
18025
18026 assert!(withheld.reports.is_empty());
18027 assert!(!withheld.covers_window);
18028 assert!(!client.dispatch.seen_trade_ids.contains(&fill.trade_id));
18029 confirm_modify_order(&pending, &acked_probe_context(&client), &raw).unwrap();
18030 let events = drain_events(&mut rx);
18031 assert_eq!(events.len(), 2);
18032 let mut events = events.into_iter();
18033
18034 let ExecutionEvent::Order(update @ OrderEventAny::Updated(_)) = events.next().unwrap()
18035 else {
18036 panic!("expected confirmed update before real fill");
18037 };
18038
18039 let ExecutionEvent::Order(delivered_fill @ OrderEventAny::Filled(_)) =
18040 events.next().unwrap()
18041 else {
18042 panic!("expected buffered real fill");
18043 };
18044
18045 let OrderEventAny::Filled(actual) = &delivered_fill else {
18046 unreachable!()
18047 };
18048
18049 assert_eq!(actual.trade_id, fill.trade_id);
18050 assert_eq!(actual.client_order_id, cloid);
18051 assert_eq!(actual.venue_order_id, fill.venue_order_id);
18052 assert_eq!(actual.last_qty, fill.last_qty);
18053 assert_eq!(actual.last_px, fill.last_px);
18054 assert_eq!(actual.commission, Some(fill.commission));
18055 assert_eq!(actual.liquidity_side, fill.liquidity_side);
18056 assert_eq!(actual.ts_event, fill.ts_event);
18057 assert!(!actual.reconciliation);
18058
18059 let still_withheld = finish_fill_reports(
18060 &client.core,
18061 &client.dispatch,
18062 BTreeSet::new(),
18063 Ok(FillSweep {
18064 reports: vec![fill.clone()],
18065 covers_window: true,
18066 withheld_instruments: AHashSet::new(),
18067 }),
18068 )
18069 .unwrap();
18070
18071 assert!(still_withheld.reports.is_empty());
18072 assert!(!still_withheld.covers_window);
18073 assert!(client.dispatch.seen_trade_ids.contains(&fill.trade_id));
18074 cache.borrow_mut().update_order(&update).unwrap();
18075 assert!(order_shape_pending(&client.core, &client.dispatch, &cloid));
18076 cache.borrow_mut().update_order(&delivered_fill).unwrap();
18077
18078 let delivered = finish_fill_reports(
18079 &client.core,
18080 &client.dispatch,
18081 BTreeSet::new(),
18082 Ok(FillSweep {
18083 reports: vec![fill.clone()],
18084 covers_window: true,
18085 withheld_instruments: AHashSet::new(),
18086 }),
18087 )
18088 .unwrap();
18089
18090 assert!(delivered.reports.is_empty());
18091 assert!(delivered.covers_window);
18092 assert_eq!(
18093 client.dispatch.mark_trade_reconciled(fill.trade_id),
18094 Some(TradeDedupSource::Live)
18095 );
18096 }
18097
18098 #[rstest]
18099 fn dispatch_lighter_trade_parse_failure_rolls_back_dedup() {
18100 let mut rig = dispatcher_rig("21");
18102 register_identity(&rig);
18103 let mut trade = dispatcher_test_trade(&rig, true);
18104 trade.timestamp = -1;
18105 let trade_id = parse_lighter_trade_id(&trade).expect("trade id parses");
18106
18107 dispatch_lighter_trade(
18108 &trade,
18109 &rig.dispatch,
18110 &rig.emitter,
18111 &rig.registry,
18112 account_id(),
18113 trader_id(),
18114 Some(TEST_ACCOUNT_INDEX_I64),
18115 UnixNanos::from(1),
18116 );
18117
18118 let events = drain_events(&mut rig.rx);
18119 assert!(
18120 !events
18121 .iter()
18122 .any(|e| matches!(e, ExecutionEvent::Order(OrderEventAny::Filled(_)))),
18123 "malformed trade must not emit a fill, was {events:?}",
18124 );
18125 assert!(
18126 rig.dispatch.mark_trade_seen(trade_id),
18127 "parse-failed trade must be re-markable (dedup rolled back)",
18128 );
18129 }
18130
18131 #[rstest]
18132 fn dispatch_lighter_trade_untracked_parse_failure_rolls_back_dedup() {
18133 let mut rig = dispatcher_rig("22");
18135 let mut trade = dispatcher_test_trade(&rig, true);
18136 trade.timestamp = -1;
18137 let trade_id = parse_lighter_trade_id(&trade).expect("trade id parses");
18138
18139 dispatch_lighter_trade(
18140 &trade,
18141 &rig.dispatch,
18142 &rig.emitter,
18143 &rig.registry,
18144 account_id(),
18145 trader_id(),
18146 Some(TEST_ACCOUNT_INDEX_I64),
18147 UnixNanos::from(1),
18148 );
18149
18150 let events = drain_events(&mut rig.rx);
18151 assert!(
18152 events.is_empty(),
18153 "untracked malformed trade must emit nothing, was {events:?}",
18154 );
18155 assert!(
18156 rig.dispatch.mark_trade_seen(trade_id),
18157 "untracked parse-failed trade must be re-markable (dedup rolled back)",
18158 );
18159 }
18160
18161 #[rstest]
18162 fn dispatch_tracked_order_event_terminal_cancel_retires_identity_and_removes_snapshot() {
18163 let mut rig = dispatcher_rig("6");
18164 register_identity(&rig);
18165 rig.dispatch.mark_accepted_emitted(rig.cloid);
18166 rig.dispatch.store_snapshot(
18167 rig.cloid,
18168 crate::websocket::dispatch::OrderShapeSnapshot {
18169 quantity: Quantity::from("0.0050"),
18170 price: Some(Price::from("2352.74")),
18171 trigger_price: None,
18172 },
18173 );
18174 let mut order = dispatcher_test_order(&rig, LighterOrderStatus::Canceled);
18175 order.filled_base_amount = Decimal::ZERO;
18176 let mut trade = dispatcher_test_trade(&rig, true);
18177 trade.bid_id = order.order_index;
18178 trade.bid_id_str = Some(order.order_id.clone());
18179 dispatch_lighter_order(
18180 &order,
18181 &rig.dispatch,
18182 &rig.emitter,
18183 &rig.registry,
18184 account_id(),
18185 trader_id(),
18186 UnixNanos::from(1),
18187 );
18188
18189 let events = drain_events(&mut rig.rx);
18190 let canceled = events
18191 .iter()
18192 .find(|e| matches!(e, ExecutionEvent::Order(OrderEventAny::Canceled(_))))
18193 .expect("expected a Canceled event");
18194 if let ExecutionEvent::Order(OrderEventAny::Canceled(e)) = canceled {
18195 assert_eq!(e.client_order_id, rig.cloid);
18196 }
18197 assert!(!rig.dispatch.order_identities.contains_key(&rig.cloid));
18198 assert!(rig.dispatch.order_identity(&rig.cloid).is_some());
18199 assert!(rig.dispatch.snapshot_for(&rig.cloid).is_none());
18200 assert!(rig.dispatch.accepted_was_emitted(&rig.cloid));
18201
18202 dispatch_lighter_trade(
18203 &trade,
18204 &rig.dispatch,
18205 &rig.emitter,
18206 &rig.registry,
18207 account_id(),
18208 trader_id(),
18209 Some(TEST_ACCOUNT_INDEX_I64),
18210 UnixNanos::from(2),
18211 );
18212 let trailing = drain_events(&mut rig.rx);
18213 assert_eq!(
18214 trailing.len(),
18215 1,
18216 "expected one trailing fill: {trailing:?}"
18217 );
18218
18219 match &trailing[0] {
18220 ExecutionEvent::Order(OrderEventAny::Filled(event)) => {
18221 assert_eq!(event.client_order_id, rig.cloid);
18222 }
18223 other => panic!("expected typed trailing OrderFilled, was {other:?}"),
18224 }
18225 }
18226
18227 #[rstest]
18228 fn dispatch_tracked_cancel_after_report_seed_skips_synthesized_accept() {
18229 let mut rig = dispatcher_rig("10");
18230 register_identity(&rig);
18231
18232 let report_order = dispatcher_test_order(&rig, LighterOrderStatus::Open);
18233 let instrument = LIGHTER_INSTRUMENT_CACHE
18234 .get(&rig.instrument_id)
18235 .expect("instrument cached");
18236 let report = parse_ws_order_status_report(
18237 &report_order,
18238 instrument.value(),
18239 account_id(),
18240 UnixNanos::from(1),
18241 )
18242 .map(|report| report.with_client_order_id(rig.cloid))
18243 .expect("report parses");
18244
18245 assert_eq!(report.order_status, OrderStatus::Accepted);
18246 rig.dispatch.seed_accepted_from_report(&report);
18247
18248 let mut cancel_order = dispatcher_test_order(&rig, LighterOrderStatus::Canceled);
18249 cancel_order.filled_base_amount = Decimal::ZERO;
18250 dispatch_lighter_order(
18251 &cancel_order,
18252 &rig.dispatch,
18253 &rig.emitter,
18254 &rig.registry,
18255 account_id(),
18256 trader_id(),
18257 UnixNanos::from(2),
18258 );
18259
18260 let events = drain_events(&mut rig.rx);
18261 assert_eq!(
18262 events.len(),
18263 1,
18264 "report-seeded cancel should emit only Canceled, was {events:?}",
18265 );
18266 assert!(
18267 !events
18268 .iter()
18269 .any(|e| matches!(e, ExecutionEvent::Order(OrderEventAny::Accepted(_)))),
18270 "typed cancel must not synthesize a second Accepted",
18271 );
18272
18273 match &events[0] {
18274 ExecutionEvent::Order(OrderEventAny::Canceled(e)) => {
18275 assert_eq!(e.client_order_id, rig.cloid);
18276 assert_eq!(e.venue_order_id, Some(VenueOrderId::new("281476929510110")));
18277 }
18278 other => panic!("expected Canceled, was {other:?}"),
18279 }
18280 }
18281
18282 #[rstest]
18283 fn dispatch_tracked_cancel_after_submitted_report_seed_skips_synthesized_accept() {
18284 let mut rig = dispatcher_rig("11");
18285 register_identity(&rig);
18286
18287 let report_order = dispatcher_test_order(&rig, LighterOrderStatus::Open);
18288 let instrument = LIGHTER_INSTRUMENT_CACHE
18289 .get(&rig.instrument_id)
18290 .expect("instrument cached");
18291 let mut report = parse_ws_order_status_report(
18292 &report_order,
18293 instrument.value(),
18294 account_id(),
18295 UnixNanos::from(1),
18296 )
18297 .map(|report| report.with_client_order_id(rig.cloid))
18298 .expect("report parses");
18299 report.order_status = OrderStatus::Submitted;
18300
18301 rig.dispatch.seed_accepted_from_report(&report);
18302 assert!(rig.dispatch.accepted_was_emitted(&rig.cloid));
18303
18304 let mut cancel_order = dispatcher_test_order(&rig, LighterOrderStatus::Canceled);
18305 cancel_order.filled_base_amount = Decimal::ZERO;
18306
18307 dispatch_lighter_order(
18308 &cancel_order,
18309 &rig.dispatch,
18310 &rig.emitter,
18311 &rig.registry,
18312 account_id(),
18313 trader_id(),
18314 UnixNanos::from(2),
18315 );
18316
18317 let events = drain_events(&mut rig.rx);
18318 assert_eq!(
18319 events.len(),
18320 1,
18321 "Cancel after Submitted report should emit only Canceled, was {events:?}",
18322 );
18323
18324 match &events[0] {
18325 ExecutionEvent::Order(OrderEventAny::Canceled(e)) => {
18326 assert_eq!(e.client_order_id, rig.cloid);
18327 assert_eq!(e.venue_order_id, Some(VenueOrderId::new("281476929510110")));
18328 }
18329 other => panic!("expected Canceled, was {other:?}"),
18330 }
18331 }
18332
18333 #[rstest]
18334 fn dispatch_tracked_order_event_accept_dedup_is_idempotent() {
18335 let mut rig = dispatcher_rig("7");
18336 register_identity(&rig);
18337 let order = dispatcher_test_order(&rig, LighterOrderStatus::Open);
18338
18339 dispatch_lighter_order(
18342 &order,
18343 &rig.dispatch,
18344 &rig.emitter,
18345 &rig.registry,
18346 account_id(),
18347 trader_id(),
18348 UnixNanos::from(1),
18349 );
18350 dispatch_lighter_order(
18351 &order,
18352 &rig.dispatch,
18353 &rig.emitter,
18354 &rig.registry,
18355 account_id(),
18356 trader_id(),
18357 UnixNanos::from(2),
18358 );
18359
18360 let events = drain_events(&mut rig.rx);
18361 let accepted_count = events
18362 .iter()
18363 .filter(|e| matches!(e, ExecutionEvent::Order(OrderEventAny::Accepted(_))))
18364 .count();
18365 assert_eq!(accepted_count, 1, "Accepted must be emitted exactly once");
18366 }
18367
18368 #[rstest]
18369 fn dispatch_lighter_order_drops_when_instrument_uncached() {
18370 let registry = Arc::new(MarketRegistry::new());
18372 let (emitter, mut rx) = dispatcher_emitter();
18373 let dispatch = WsDispatchState::new();
18374 let cloid = ClientOrderId::new("CLOID-MISSING");
18375 dispatch.register_order_identity(
18376 cloid,
18377 OrderIdentity::new(
18378 InstrumentId::from("MISSING-PERP.LIGHTER"),
18379 strategy_id(),
18380 OrderSide::Buy,
18381 OrderType::Limit,
18382 1,
18383 ),
18384 );
18385 let mut order = LighterOrder {
18386 order_index: 1,
18387 client_order_index: 1,
18388 order_id: "1".to_string(),
18389 client_order_id: "1".to_string(),
18390 market_index: 999, owner_account_index: TEST_ACCOUNT_INDEX_I64,
18392 initial_base_amount: Decimal::ZERO,
18393 price: Decimal::ZERO,
18394 nonce: 0,
18395 remaining_base_amount: Decimal::ZERO,
18396 is_ask: false,
18397 base_size: 0,
18398 base_price: 0,
18399 filled_base_amount: Decimal::ZERO,
18400 filled_quote_amount: Decimal::ZERO,
18401 side: Some(LighterOrderSide::Buy),
18402 order_type: LighterOrderKind::Limit,
18403 time_in_force: LighterOrderTimeInForce::GoodTillTime,
18404 reduce_only: false,
18405 trigger_price: Decimal::ZERO,
18406 order_expiry: 0,
18407 status: LighterOrderStatus::Open,
18408 trigger_status: LighterTriggerStatus::Na,
18409 trigger_time: 0,
18410 parent_order_index: 0,
18411 parent_order_id: "0".to_string(),
18412 to_trigger_order_id_0: "0".to_string(),
18413 to_trigger_order_id_1: "0".to_string(),
18414 to_cancel_order_id_0: "0".to_string(),
18415 integrator_fee_collector_index: "0".to_string(),
18416 integrator_taker_fee: Decimal::ZERO,
18417 integrator_maker_fee: Decimal::ZERO,
18418 block_height: 0,
18419 timestamp: 0,
18420 created_at: 0,
18421 updated_at: 0,
18422 transaction_time: 0,
18423 };
18424 order.client_order_id = "1".to_string();
18425
18426 dispatch_lighter_order(
18427 &order,
18428 &dispatch,
18429 &emitter,
18430 ®istry,
18431 account_id(),
18432 trader_id(),
18433 UnixNanos::from(1),
18434 );
18435
18436 let events = drain_events(&mut rx);
18437 assert!(
18438 events.is_empty(),
18439 "no event for uncached instrument, was {events:?}"
18440 );
18441 assert!(dispatch.order_identities.contains_key(&cloid));
18442 }
18443
18444 #[rstest]
18445 fn dispatch_lighter_trade_filters_non_account_trades_defensively() {
18446 let mut rig = dispatcher_rig("8");
18447 let mut trade = dispatcher_test_trade(&rig, true);
18452 trade.bid_account_id = 91_249;
18453 trade.ask_account_id = 91_250;
18454
18455 dispatch_lighter_trade(
18456 &trade,
18457 &rig.dispatch,
18458 &rig.emitter,
18459 &rig.registry,
18460 account_id(),
18461 trader_id(),
18462 Some(TEST_ACCOUNT_INDEX_I64),
18463 UnixNanos::from(1),
18464 );
18465
18466 let events = drain_events(&mut rig.rx);
18467 assert!(
18468 events.is_empty(),
18469 "foreign trade must produce no event, was {events:?}"
18470 );
18471 assert!(
18472 !rig.dispatch
18473 .seen_trade_ids
18474 .contains(&TradeId::new("19209006902"),)
18475 );
18476 }
18477
18478 #[rstest]
18479 fn register_cloid_in_submit_path_uses_probed_index_on_collision() {
18480 let dispatch = WsDispatchState::new();
18485 let cloid = ClientOrderId::new("PROBE-CLOID");
18486 let derived = dispatch.derive_client_order_index(&cloid);
18487
18488 let intruder = ClientOrderId::new("INTRUDER");
18489 dispatch.cloid_map.insert(derived, intruder);
18490
18491 let chosen = dispatch.register_cloid(derived, cloid).unwrap();
18492
18493 assert_ne!(chosen, derived);
18494 assert_eq!(
18495 dispatch.cloid_map.get(&derived).map(|e| *e.value()),
18496 Some(intruder),
18497 );
18498 assert_eq!(
18499 dispatch.cloid_map.get(&chosen).map(|e| *e.value()),
18500 Some(cloid),
18501 );
18502 }
18503
18504 #[rstest]
18505 fn reconciliation_restores_active_collision_probed_identity() {
18506 let (client, cache, _rx) = create_execution_client();
18507 let instrument_id = register_test_instrument(&client, &cache);
18508 let mut factory = test_order_factory();
18509 let order = test_limit_order(&mut factory, instrument_id, "O-RECON-ACTIVE");
18510 let cloid = order.client_order_id();
18511 let venue_order_id = VenueOrderId::from("281476929510110");
18512 cache_accepted_order(&cache, order, venue_order_id, None);
18513 let (base_index, client_order_index) = forced_probed_index(cloid);
18514 let raw =
18515 reconciliation_raw_order(client_order_index, venue_order_id, LighterOrderStatus::Open);
18516
18517 restore_reconciled_order(
18518 &client.core,
18519 &client.dispatch,
18520 VenueOrderId::new(raw.order_id.as_str()),
18521 raw.client_order_index,
18522 false,
18523 );
18524 let report = parse_http_order_to_report(
18525 &raw,
18526 &client.registry,
18527 account_id(),
18528 UnixNanos::from(1),
18529 &client.dispatch,
18530 )
18531 .unwrap();
18532 let translated = client.dispatch.translate_order_cloid(report);
18533 let replacement = ClientOrderId::from("O-RECON-ACTIVE-REPLACEMENT");
18534 let replacement_index = client
18535 .dispatch
18536 .register_cloid(client_order_index, replacement)
18537 .unwrap();
18538 let wrong_venue_order_id = VenueOrderId::from("281476929510111");
18539 let wrong_raw = reconciliation_raw_order(
18540 client_order_index,
18541 wrong_venue_order_id,
18542 LighterOrderStatus::Open,
18543 );
18544 let wrong_report = parse_http_order_to_report(
18545 &wrong_raw,
18546 &client.registry,
18547 account_id(),
18548 UnixNanos::from(2),
18549 &client.dispatch,
18550 )
18551 .unwrap();
18552 let wrong_translated = client.dispatch.translate_order_cloid(wrong_report);
18553
18554 assert_ne!(client_order_index, base_index);
18555 assert_eq!(
18556 client.dispatch.client_order_index(&cloid),
18557 Some(client_order_index),
18558 );
18559 assert_eq!(
18560 client
18561 .dispatch
18562 .resolve_client_order_index(client_order_index),
18563 Some(cloid),
18564 );
18565 assert_eq!(
18566 client.dispatch.lookup_venue_order_id(&cloid),
18567 Some(venue_order_id),
18568 );
18569 assert!(client.dispatch.order_identities.contains_key(&cloid));
18570 assert!(client.dispatch.accepted_was_emitted(&cloid));
18571 assert_eq!(translated.client_order_id, Some(cloid));
18572 assert_eq!(translated.venue_order_id, venue_order_id);
18573 assert_ne!(replacement_index, client_order_index);
18574 assert_eq!(
18575 client
18576 .dispatch
18577 .resolve_client_order_index(replacement_index),
18578 Some(replacement),
18579 );
18580 assert_eq!(
18581 wrong_translated.client_order_id,
18582 Some(ClientOrderId::new(wrong_venue_order_id.as_str())),
18583 );
18584 assert_eq!(wrong_translated.venue_order_id, wrong_venue_order_id);
18585 }
18586
18587 #[rstest]
18588 fn reconciliation_retires_stale_active_cache_from_terminal_report() {
18589 let (client, cache, _rx) = create_execution_client();
18590 let instrument_id = register_test_instrument(&client, &cache);
18591 let mut factory = test_order_factory();
18592 let order = test_limit_order(&mut factory, instrument_id, "O-RECON-FILLED");
18593 let cloid = order.client_order_id();
18594 let venue_order_id = VenueOrderId::from("281476929510119");
18595 cache_accepted_order(&cache, order, venue_order_id, None);
18596 let (_, client_order_index) = forced_probed_index(cloid);
18597 let raw = reconciliation_raw_order(
18598 client_order_index,
18599 venue_order_id,
18600 LighterOrderStatus::Filled,
18601 );
18602
18603 restore_reconciled_order(
18604 &client.core,
18605 &client.dispatch,
18606 VenueOrderId::new(raw.order_id.as_str()),
18607 raw.client_order_index,
18608 true,
18609 );
18610 let fill = client
18611 .dispatch
18612 .translate_fill_cloid(reconciliation_fill_report(
18613 instrument_id,
18614 client_order_index,
18615 venue_order_id,
18616 "19209006919",
18617 ));
18618
18619 assert!(!client.dispatch.cloid_map.contains_key(&client_order_index));
18620 assert!(!client.dispatch.order_identities.contains_key(&cloid));
18621 assert_eq!(
18622 client.dispatch.client_order_index(&cloid),
18623 Some(client_order_index),
18624 );
18625 assert_eq!(fill.client_order_id, Some(cloid));
18626 assert_eq!(fill.venue_order_id, venue_order_id);
18627 }
18628
18629 #[rstest]
18630 fn reconciliation_restores_reused_retired_index_for_fill_translation() {
18631 let (client, cache, _rx) = create_execution_client();
18632 let instrument_id = register_test_instrument(&client, &cache);
18633 let mut factory = test_order_factory();
18634 let first_order = test_limit_order(&mut factory, instrument_id, "O-RECON-RETIRED-A");
18635 let second_order = test_limit_order(&mut factory, instrument_id, "O-RECON-RETIRED-B");
18636 let first_cloid = first_order.client_order_id();
18637 let second_cloid = second_order.client_order_id();
18638 let first_venue_order_id = VenueOrderId::from("281476929510120");
18639 let second_venue_order_id = VenueOrderId::from("281476929510121");
18640 cache_canceled_order(&cache, first_order, first_venue_order_id);
18641 cache_canceled_order(&cache, second_order, second_venue_order_id);
18642 let (_, client_order_index) = forced_probed_index(first_cloid);
18643 let first_raw = reconciliation_raw_order(
18644 client_order_index,
18645 first_venue_order_id,
18646 LighterOrderStatus::Canceled,
18647 );
18648 let second_raw = reconciliation_raw_order(
18649 client_order_index,
18650 second_venue_order_id,
18651 LighterOrderStatus::Canceled,
18652 );
18653
18654 restore_reconciled_order(
18655 &client.core,
18656 &client.dispatch,
18657 VenueOrderId::new(first_raw.order_id.as_str()),
18658 first_raw.client_order_index,
18659 true,
18660 );
18661 restore_reconciled_order(
18662 &client.core,
18663 &client.dispatch,
18664 VenueOrderId::new(second_raw.order_id.as_str()),
18665 second_raw.client_order_index,
18666 true,
18667 );
18668 let first_fill = client
18669 .dispatch
18670 .translate_fill_cloid(reconciliation_fill_report(
18671 instrument_id,
18672 client_order_index,
18673 first_venue_order_id,
18674 "19209006920",
18675 ));
18676 let second_fill = client
18677 .dispatch
18678 .translate_fill_cloid(reconciliation_fill_report(
18679 instrument_id,
18680 client_order_index,
18681 second_venue_order_id,
18682 "19209006921",
18683 ));
18684 let replacement = ClientOrderId::from("O-RECON-RETIRED-REPLACEMENT");
18685 let replacement_index = client
18686 .dispatch
18687 .register_cloid(client_order_index, replacement)
18688 .unwrap();
18689
18690 assert!(!client.dispatch.cloid_map.contains_key(&client_order_index));
18691 assert!(!client.dispatch.order_identities.contains_key(&first_cloid));
18692 assert!(!client.dispatch.order_identities.contains_key(&second_cloid));
18693 assert_eq!(
18694 client.dispatch.client_order_index(&first_cloid),
18695 Some(client_order_index),
18696 );
18697 assert_eq!(
18698 client.dispatch.client_order_index(&second_cloid),
18699 Some(client_order_index),
18700 );
18701 assert_eq!(
18702 client
18703 .dispatch
18704 .resolve_client_order_index(client_order_index),
18705 None,
18706 );
18707 assert_eq!(first_fill.client_order_id, Some(first_cloid));
18708 assert_eq!(first_fill.venue_order_id, first_venue_order_id);
18709 assert_eq!(second_fill.client_order_id, Some(second_cloid));
18710 assert_eq!(second_fill.venue_order_id, second_venue_order_id);
18711 assert_ne!(replacement_index, client_order_index);
18712 assert_eq!(
18713 client
18714 .dispatch
18715 .resolve_client_order_index(replacement_index),
18716 Some(replacement),
18717 );
18718 }
18719
18720 #[rstest]
18721 fn reconciliation_restores_triggered_marker() {
18722 let (client, cache, _rx) = create_execution_client();
18723 let instrument_id = register_test_instrument(&client, &cache);
18724 let mut factory = test_order_factory();
18725 let order = factory.stop_limit(
18726 instrument_id,
18727 OrderSide::Sell,
18728 Quantity::from("0.1000"),
18729 Price::from("2290.00"),
18730 Price::from("2300.00"),
18731 None,
18732 Some(TimeInForce::Gtc),
18733 None,
18734 Some(false),
18735 Some(false),
18736 Some(false),
18737 None,
18738 None,
18739 None,
18740 None,
18741 None,
18742 None,
18743 Some(ClientOrderId::from("O-RECON-TRIGGERED")),
18744 );
18745 let cloid = order.client_order_id();
18746 let venue_order_id = VenueOrderId::from("281476929510124");
18747 cache_accepted_order(&cache, order, venue_order_id, None);
18748 let triggered = OrderEventAny::Triggered(OrderTriggered::new(
18749 trader_id(),
18750 strategy_id(),
18751 instrument_id,
18752 cloid,
18753 UUID4::new(),
18754 UnixNanos::from(1),
18755 UnixNanos::from(2),
18756 false,
18757 Some(venue_order_id),
18758 Some(account_id()),
18759 ));
18760 cache.borrow_mut().update_order(&triggered).unwrap();
18761 let (_, client_order_index) = forced_probed_index(cloid);
18762 let raw =
18763 reconciliation_raw_order(client_order_index, venue_order_id, LighterOrderStatus::Open);
18764
18765 restore_reconciled_order(
18766 &client.core,
18767 &client.dispatch,
18768 VenueOrderId::new(raw.order_id.as_str()),
18769 raw.client_order_index,
18770 false,
18771 );
18772
18773 assert!(client.dispatch.order_identities.contains_key(&cloid));
18774 assert!(client.dispatch.accepted_was_emitted(&cloid));
18775 assert!(client.dispatch.triggered_was_emitted(&cloid));
18776 }
18777
18778 #[rstest]
18779 fn reconciliation_routes_reused_index_by_exact_venue_order_id() {
18780 let (client, cache, _rx) = create_execution_client();
18781 let instrument_id = register_test_instrument(&client, &cache);
18782 let mut factory = test_order_factory();
18783 let active_order = test_limit_order(&mut factory, instrument_id, "O-RECON-LIVE-ACTIVE");
18784 let retired_order = test_limit_order(&mut factory, instrument_id, "O-RECON-LIVE-RETIRED");
18785 let active_cloid = active_order.client_order_id();
18786 let retired_cloid = retired_order.client_order_id();
18787 let active_venue_order_id = VenueOrderId::from("281476929510125");
18788 let retired_venue_order_id = VenueOrderId::from("281476929510126");
18789 cache_accepted_order(
18790 &cache,
18791 active_order,
18792 active_venue_order_id,
18793 Some(client_id()),
18794 );
18795 cache_canceled_order(&cache, retired_order, retired_venue_order_id);
18796 let (_, client_order_index) = forced_probed_index(active_cloid);
18797 let active_raw = reconciliation_raw_order(
18798 client_order_index,
18799 active_venue_order_id,
18800 LighterOrderStatus::Open,
18801 );
18802 let retired_raw = reconciliation_raw_order(
18803 client_order_index,
18804 retired_venue_order_id,
18805 LighterOrderStatus::Canceled,
18806 );
18807 let raw_client_id = client_order_index.to_string();
18808
18809 restore_reconciled_order(
18810 &client.core,
18811 &client.dispatch,
18812 VenueOrderId::new(active_raw.order_id.as_str()),
18813 active_raw.client_order_index,
18814 false,
18815 );
18816 restore_reconciled_order(
18817 &client.core,
18818 &client.dispatch,
18819 VenueOrderId::new(retired_raw.order_id.as_str()),
18820 retired_raw.client_order_index,
18821 true,
18822 );
18823
18824 assert_eq!(
18825 client
18826 .dispatch
18827 .resolve_live_order_cloid(&raw_client_id, active_venue_order_id,),
18828 Some(active_cloid),
18829 );
18830 assert_eq!(
18831 client
18832 .dispatch
18833 .resolve_live_order_cloid(&raw_client_id, retired_venue_order_id,),
18834 Some(retired_cloid),
18835 );
18836 assert_eq!(
18837 client
18838 .dispatch
18839 .resolve_live_trade_cloid(&raw_client_id, active_venue_order_id),
18840 Some(active_cloid),
18841 );
18842 assert_eq!(
18843 client
18844 .dispatch
18845 .resolve_live_trade_cloid(&raw_client_id, retired_venue_order_id),
18846 Some(retired_cloid),
18847 );
18848 }
18849
18850 #[rstest]
18851 fn reconciliation_does_not_restore_from_numeric_client_index_alone() {
18852 let (client, cache, _rx) = create_execution_client();
18853 let instrument_id = register_test_instrument(&client, &cache);
18854 let mut factory = test_order_factory();
18855 let order = test_limit_order(&mut factory, instrument_id, "O-RECON-NUMERIC-ONLY");
18856 let cloid = order.client_order_id();
18857 let client_order_index = client.dispatch.derive_client_order_index(&cloid);
18858 let venue_order_id = VenueOrderId::from("281476929510130");
18859 cache_order(&cache, order);
18860 let raw =
18861 reconciliation_raw_order(client_order_index, venue_order_id, LighterOrderStatus::Open);
18862
18863 restore_reconciled_order(
18864 &client.core,
18865 &client.dispatch,
18866 VenueOrderId::new(raw.order_id.as_str()),
18867 raw.client_order_index,
18868 false,
18869 );
18870 let report = parse_http_order_to_report(
18871 &raw,
18872 &client.registry,
18873 account_id(),
18874 UnixNanos::from(1),
18875 &client.dispatch,
18876 )
18877 .unwrap();
18878 let translated = client.dispatch.translate_order_cloid(report);
18879
18880 assert_eq!(client.dispatch.client_order_index(&cloid), None);
18881 assert_eq!(
18882 client
18883 .dispatch
18884 .resolve_client_order_index(client_order_index),
18885 None,
18886 );
18887 assert!(!client.dispatch.order_identities.contains_key(&cloid));
18888 assert_eq!(
18889 translated.client_order_id,
18890 Some(ClientOrderId::new(venue_order_id.as_str())),
18891 );
18892 assert_eq!(translated.venue_order_id, venue_order_id);
18893 }
18894
18895 #[rstest]
18896 #[case::known_partner(true)]
18897 #[case::external_partner(false)]
18898 fn lookup_after_restart_preserves_collision_partner(#[case] partner_cached: bool) {
18899 let (client, cache, _rx) = create_execution_client();
18900 let instrument_id = register_test_instrument(&client, &cache);
18901 let mut factory = test_order_factory();
18902 let order = test_limit_order(&mut factory, instrument_id, "O-UNACKNOWLEDGED-PROBED");
18903 let cloid = order.client_order_id();
18904 let partner = test_limit_order(&mut factory, instrument_id, "O-COLLISION-PARTNER");
18905 let partner_cloid = partner.client_order_id();
18906 let venue_order_id = VenueOrderId::from("281476929510140");
18907 let source = WsDispatchState::new();
18908 let base_index = source.derive_client_order_index(&cloid);
18909 source.register_cloid(base_index, partner_cloid).unwrap();
18910 let selected_index = source.register_cloid(base_index, cloid).unwrap();
18911 cache_order(&cache, order);
18912
18913 let submitted = OrderSubmitted::new(
18914 trader_id(),
18915 strategy_id(),
18916 instrument_id,
18917 cloid,
18918 account_id(),
18919 UUID4::new(),
18920 UnixNanos::from(1),
18921 UnixNanos::from(2),
18922 );
18923 cache
18924 .borrow_mut()
18925 .update_order(&OrderEventAny::Submitted(submitted))
18926 .unwrap();
18927
18928 if partner_cached {
18929 cache_accepted_order(&cache, partner, venue_order_id, Some(client_id()));
18930 restore_reconciled_order(
18931 &client.core,
18932 &client.dispatch,
18933 venue_order_id,
18934 base_index,
18935 false,
18936 );
18937 }
18938
18939 let raw = reconciliation_raw_order(base_index, venue_order_id, LighterOrderStatus::Open);
18940 let lookup = OrderReportLookup::new(
18941 &client.registry,
18942 &client.dispatch,
18943 Some(instrument_id),
18944 Some(&cloid),
18945 None,
18946 )
18947 .unwrap();
18948 let parsed = lookup
18949 .parse(
18950 &raw,
18951 &client.registry,
18952 account_id(),
18953 UnixNanos::from(3),
18954 &client.dispatch,
18955 )
18956 .unwrap();
18957
18958 let expected_cloid = if partner_cached {
18959 partner_cloid
18960 } else {
18961 ClientOrderId::new(venue_order_id.as_str())
18962 };
18963
18964 let expected = parsed.0.clone().with_client_order_id(expected_cloid);
18965 let report = lookup.finish(&client.dispatch, parsed);
18966 let cached = cache.borrow().order_owned(&cloid).unwrap();
18967
18968 assert_eq!(selected_index, base_index + 1);
18969 assert_eq!(
18970 serde_json::to_value(&report).unwrap(),
18971 serde_json::to_value(&expected).unwrap()
18972 );
18973 assert_eq!(cached.status(), OrderStatus::Submitted);
18974 assert_eq!(cached.venue_order_id(), None);
18975 assert_eq!(client.dispatch.client_order_index(&cloid), None);
18976 }
18977
18978 fn forced_probed_index(cloid: ClientOrderId) -> (i64, i64) {
18979 let source = WsDispatchState::new();
18980 let base_index = source.derive_client_order_index(&cloid);
18981 source
18982 .cloid_map
18983 .insert(base_index, ClientOrderId::from("O-FORCED-COLLISION"));
18984 let client_order_index = source.register_cloid(base_index, cloid).unwrap();
18985 (base_index, client_order_index)
18986 }
18987
18988 fn reconciliation_raw_order(
18989 client_order_index: i64,
18990 venue_order_id: VenueOrderId,
18991 status: LighterOrderStatus,
18992 ) -> LighterOrder {
18993 let rig = dispatcher_rig("RECONCILIATION-RAW");
18994 let mut raw = dispatcher_test_order(&rig, status);
18995 raw.order_index = venue_order_id.as_str().parse().unwrap();
18996 raw.order_id = venue_order_id.to_string();
18997 raw.client_order_index = client_order_index;
18998 raw.client_order_id = client_order_index.to_string();
18999 raw
19000 }
19001
19002 fn cache_canceled_order(
19003 cache: &Rc<RefCell<Cache>>,
19004 order: OrderAny,
19005 venue_order_id: VenueOrderId,
19006 ) {
19007 let (instrument_id, client_order_id) =
19008 cache_accepted_order(cache, order, venue_order_id, Some(client_id()));
19009 let canceled = OrderEventAny::Canceled(OrderCanceled::new(
19010 trader_id(),
19011 strategy_id(),
19012 instrument_id,
19013 client_order_id,
19014 UUID4::new(),
19015 UnixNanos::from(1),
19016 UnixNanos::from(2),
19017 false,
19018 Some(venue_order_id),
19019 Some(account_id()),
19020 None,
19021 ));
19022 cache.borrow_mut().update_order(&canceled).unwrap();
19023 }
19024
19025 fn reconciliation_fill_report(
19026 instrument_id: InstrumentId,
19027 client_order_index: i64,
19028 venue_order_id: VenueOrderId,
19029 trade_id: &str,
19030 ) -> FillReport {
19031 FillReport::new(
19032 account_id(),
19033 instrument_id,
19034 venue_order_id,
19035 TradeId::new(trade_id),
19036 OrderSide::Buy,
19037 Quantity::from("0.1336"),
19038 Price::from("2352.73"),
19039 Money::from("0.000196 USDC"),
19040 LiquiditySide::Taker,
19041 Some(ClientOrderId::new(client_order_index.to_string())),
19042 None,
19043 UnixNanos::from(1),
19044 UnixNanos::from(2),
19045 Some(UUID4::new()),
19046 )
19047 }
19048
19049 #[rstest]
19050 fn dispatch_lighter_order_seeds_snapshot_after_synthesized_accept() {
19051 let mut rig = dispatcher_rig("9");
19056 register_identity(&rig);
19057
19058 let order = dispatcher_test_order(&rig, LighterOrderStatus::Open);
19059 let mut trade = dispatcher_test_trade(&rig, true);
19060 trade.bid_id = order.order_index;
19061 trade.bid_id_str = Some(order.order_id.clone());
19062 dispatch_lighter_trade(
19063 &trade,
19064 &rig.dispatch,
19065 &rig.emitter,
19066 &rig.registry,
19067 account_id(),
19068 trader_id(),
19069 Some(TEST_ACCOUNT_INDEX_I64),
19070 UnixNanos::from(1),
19071 );
19072 assert_eq!(drain_events(&mut rig.rx).len(), 2);
19073 assert!(rig.dispatch.accepted_was_emitted(&rig.cloid));
19074 assert!(
19075 rig.dispatch.snapshot_for(&rig.cloid).is_none(),
19076 "synthesized Accept has no snapshot until the Open frame seeds one",
19077 );
19078
19079 dispatch_lighter_order(
19083 &order,
19084 &rig.dispatch,
19085 &rig.emitter,
19086 &rig.registry,
19087 account_id(),
19088 trader_id(),
19089 UnixNanos::from(2),
19090 );
19091 assert!(
19092 rig.dispatch.snapshot_for(&rig.cloid).is_some(),
19093 "Open frame after synthesized accept must seed the snapshot",
19094 );
19095
19096 let mut modified = order;
19098 modified.price = Decimal::from_str("2400.00").unwrap();
19099 dispatch_lighter_order(
19100 &modified,
19101 &rig.dispatch,
19102 &rig.emitter,
19103 &rig.registry,
19104 account_id(),
19105 trader_id(),
19106 UnixNanos::from(3),
19107 );
19108 let events = drain_events(&mut rig.rx);
19109 let updated = events
19110 .iter()
19111 .find(|e| matches!(e, ExecutionEvent::Order(OrderEventAny::Updated(_))));
19112 assert!(
19113 updated.is_some(),
19114 "real modify must produce Updated, events={events:?}"
19115 );
19116 }
19117
19118 fn enqueue_create(client: &LighterExecutionClient, order: &OrderAny, nonce: i64) -> i64 {
19119 let client_order_index = client
19120 .dispatch
19121 .derive_client_order_index(&order.client_order_id());
19122 client
19123 .dispatch
19124 .register_cloid(client_order_index, order.client_order_id())
19125 .unwrap();
19126 client.dispatch.register_order_identity(
19127 order.client_order_id(),
19128 OrderIdentity::new(
19129 order.instrument_id(),
19130 order.strategy_id(),
19131 order.order_side(),
19132 order.order_type(),
19133 client_order_index,
19134 ),
19135 );
19136 let now = UnixNanos::from(1_000_000_000);
19137 client.dispatch.enqueue_pending_sendtx(PendingSendTx {
19138 batch_id: None,
19139 connection_epoch: 0,
19140 kind: PendingSendTxKind::Create {
19141 order: Box::new(order.clone()),
19142 client_order_index,
19143 },
19144 submitted_at: now,
19145 nonce,
19146 api_key_index: TEST_API_KEY_INDEX,
19147 tx_hash: format!("hash{nonce:02x}"),
19148 });
19149 client.dispatch.nonce_manager.refresh(
19150 TEST_ACCOUNT_INDEX_I64,
19151 TEST_API_KEY_INDEX,
19152 nonce + 1,
19153 );
19154 let _ = client
19155 .dispatch
19156 .nonce_manager
19157 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX);
19158 client_order_index
19159 }
19160
19161 fn enqueue_other(client: &LighterExecutionClient, nonce: i64) {
19162 client.dispatch.enqueue_pending_sendtx(PendingSendTx {
19163 batch_id: None,
19164 connection_epoch: 0,
19165 kind: PendingSendTxKind::Other,
19166 submitted_at: UnixNanos::from(1_000_000_000),
19167 nonce,
19168 api_key_index: TEST_API_KEY_INDEX,
19169 tx_hash: format!("hash{nonce:02x}"),
19170 });
19171 }
19172
19173 #[rstest]
19174 #[case(serde_json::json!({"self_trade_behavior_mode": "reduce"}), LighterSelfTradeBehaviorMode::Reduce, LighterSelfTradeEqualityMode::AccountIndex)]
19175 #[case(serde_json::json!({"self_trade_equality_mode": "master_account_index"}), LighterSelfTradeBehaviorMode::ExpireMaker, LighterSelfTradeEqualityMode::MasterAccountIndex)]
19176 #[case(serde_json::json!({"market_order_slippage_bps": 37}), LighterSelfTradeBehaviorMode::ExpireMaker, LighterSelfTradeEqualityMode::AccountIndex)]
19177 fn resolve_self_trade_partial_params(
19178 #[case] value: serde_json::Value,
19179 #[case] behavior: LighterSelfTradeBehaviorMode,
19180 #[case] equality: LighterSelfTradeEqualityMode,
19181 ) {
19182 let (client, _cache, _rx) = create_execution_client();
19183 let params: Params = serde_json::from_value(value).unwrap();
19184
19185 let attrs = client.resolve_create_attributes(Some(¶ms)).unwrap();
19186
19187 assert_eq!(
19188 attrs,
19189 L2TxAttributes {
19190 self_trade_behavior_mode: Some(behavior),
19191 self_trade_equality_mode: Some(equality),
19192 ..Default::default()
19193 }
19194 );
19195 }
19196
19197 fn enqueue_cancel(
19198 client: &LighterExecutionClient,
19199 instrument_id: InstrumentId,
19200 client_order_id: ClientOrderId,
19201 venue_order_id: VenueOrderId,
19202 nonce: i64,
19203 ) {
19204 client
19205 .dispatch
19206 .set_pending_order_action(client_order_id, PendingOrderAction::Cancel);
19207 client
19208 .dispatch
19209 .set_pending_cancel_nonce(&client_order_id, 0, nonce, true);
19210 client.dispatch.enqueue_pending_sendtx(PendingSendTx {
19211 batch_id: None,
19212 connection_epoch: 0,
19213 kind: PendingSendTxKind::Cancel {
19214 emit_cancel_rejected: true,
19215 strategy_id: strategy_id(),
19216 instrument_id,
19217 client_order_id,
19218 venue_order_id: Some(venue_order_id),
19219 },
19220 submitted_at: UnixNanos::from(1_000_000_000),
19221 nonce,
19222 api_key_index: TEST_API_KEY_INDEX,
19223 tx_hash: format!("hash{nonce:02x}"),
19224 });
19225 }
19226
19227 fn enqueue_modify(
19228 client: &LighterExecutionClient,
19229 instrument_id: InstrumentId,
19230 client_order_id: ClientOrderId,
19231 venue_order_id: VenueOrderId,
19232 nonce: i64,
19233 ) {
19234 let command = ModifyOrder::new(
19235 trader_id(),
19236 Some(client_id()),
19237 strategy_id(),
19238 instrument_id,
19239 client_order_id,
19240 Some(venue_order_id),
19241 Some(Quantity::from("0.2000")),
19242 Some(Price::from("2362.00")),
19243 None,
19244 UUID4::new(),
19245 UnixNanos::default(),
19246 None,
19247 None,
19248 );
19249 client.dispatch.set_pending_modify_action(&command);
19250 client.dispatch.set_pending_modify_shape(
19251 &command,
19252 OrderShapeSnapshot {
19253 quantity: command.quantity.unwrap(),
19254 price: command.price,
19255 trigger_price: command.trigger_price,
19256 },
19257 );
19258
19259 client
19260 .dispatch
19261 .set_pending_modify_nonce(&client_order_id, command.command_id, 0, nonce);
19262 client.dispatch.enqueue_pending_sendtx(PendingSendTx {
19263 batch_id: None,
19264 connection_epoch: 0,
19265 kind: PendingSendTxKind::Modify {
19266 strategy_id: strategy_id(),
19267 instrument_id,
19268 client_order_id,
19269 venue_order_id: Some(venue_order_id),
19270 },
19271 submitted_at: UnixNanos::from(1_000_000_000),
19272 nonce,
19273 api_key_index: TEST_API_KEY_INDEX,
19274 tx_hash: format!("hash{nonce:02x}"),
19275 });
19276 }
19277
19278 fn acked_probe_context(client: &LighterExecutionClient) -> AckedOrderProbeContext {
19279 AckedOrderProbeContext {
19280 http_client: client.http_client.clone(),
19281 registry: Arc::clone(&client.registry),
19282 credential: test_credential(),
19283 dispatch: client.dispatch.clone(),
19284 account_id: client.core.account_id,
19285 clock: client.clock,
19286 emitter: client.emitter.clone(),
19287 connection_epoch: client.ws_client.connection_epoch_atomic(),
19288 cancellation_token: client.cancellation_token.clone(),
19289 pending_tasks: client.pending_tasks.spawner().unwrap(),
19290 }
19291 }
19292
19293 fn failed_modify_tx(pending: &PendingSendTx) -> LighterTx {
19294 LighterTx {
19295 code: 200,
19296 message: None,
19297 hash: pending.tx_hash.clone(),
19298 tx_type: LighterTxType::ModifyOrder as u8,
19299 info: String::new(),
19300 event_info: serde_json::json!({"ae":"invalid order price"}).to_string(),
19301 status: LighterTxStatus::Failed,
19302 account_index: TEST_ACCOUNT_INDEX_I64,
19303 nonce: pending.nonce,
19304 api_key_index: pending.api_key_index,
19305 }
19306 }
19307
19308 #[rstest]
19309 #[case::hash("hash")]
19310 #[case::type_mismatch("type")]
19311 #[case::account("account")]
19312 #[case::key("key")]
19313 #[case::nonce("nonce")]
19314 #[tokio::test]
19315 async fn acknowledged_modify_requires_exact_transaction_identity(#[case] mismatch: &str) {
19316 let (client, cache, mut rx) = create_execution_client();
19317 let instrument = register_test_instrument(&client, &cache);
19318 let id = ClientOrderId::from("MODIFY-IDENTITY");
19319 enqueue_modify(&client, instrument, id, VenueOrderId::from("789"), 23);
19320 let pending = client.dispatch.pop_pending_sendtx_head().unwrap();
19321 let mut tx = failed_modify_tx(&pending);
19322 match mismatch {
19323 "hash" => tx.hash = "different".to_string(),
19324 "type" => tx.tx_type = LighterTxType::CancelOrder as u8,
19325 "account" => tx.account_index += 1,
19326 "key" => tx.api_key_index += 1,
19327 "nonce" => tx.nonce += 1,
19328 _ => unreachable!(),
19329 }
19330
19331 let result = resolve_acked_action(&pending, &acked_probe_context(&client), tx).await;
19332 assert_eq!(
19333 result.unwrap_err().to_string(),
19334 "Lighter transaction lookup did not match acknowledged action identity"
19335 );
19336 assert_eq!(client.dispatch.pending_modify_nonce(&id), Some((0, 23)));
19337 assert_eq!(
19338 client.dispatch.pending_order_action(&id),
19339 Some(PendingOrderAction::Modify)
19340 );
19341 assert!(rx.try_recv().is_err());
19342 }
19343
19344 #[rstest]
19345 #[case::failed_status(LighterTxStatus::Failed, "", true)]
19346 #[case::malformed_failed_status(LighterTxStatus::Failed, "invalid json", true)]
19347 #[case::application_error(LighterTxStatus::Executed, "application", true)]
19348 #[case::pending(LighterTxStatus::Pending, "", false)]
19349 #[case::pending_final(LighterTxStatus::PendingFinal, "", false)]
19350 #[tokio::test]
19351 async fn acknowledged_modify_outcomes_consume_only_matching_action(
19352 #[case] status: LighterTxStatus,
19353 #[case] event_info: &str,
19354 #[case] failed: bool,
19355 ) {
19356 let (client, cache, mut rx) = create_execution_client();
19357 let instrument = register_test_instrument(&client, &cache);
19358 let id = ClientOrderId::from("MODIFY-OUTCOME");
19359 let venue = VenueOrderId::from("789");
19360 let nonce = client
19361 .dispatch
19362 .nonce_manager
19363 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
19364 .unwrap();
19365 enqueue_modify(&client, instrument, id, venue, nonce);
19366 let pending = client.dispatch.pop_pending_sendtx_head().unwrap();
19367 let context = acked_probe_context(&client);
19368 let mut tx = failed_modify_tx(&pending);
19369 tx.status = status;
19370
19371 tx.event_info = if event_info == "application" {
19372 serde_json::json!({"ae":"invalid order price"}).to_string()
19373 } else {
19374 event_info.to_string()
19375 };
19376
19377 let resolved = resolve_acked_action(&pending, &context, tx.clone())
19378 .await
19379 .unwrap();
19380 assert_eq!(resolved, failed);
19381
19382 if failed {
19383 let OrderEventAny::ModifyRejected(event) = recv_order_event(&mut rx).await else {
19384 panic!("expected ModifyRejected")
19385 };
19386
19387 assert_eq!(event.client_order_id, id);
19388 assert_eq!(event.venue_order_id, Some(venue));
19389 assert_eq!(event.instrument_id, instrument);
19390 assert_eq!(event.strategy_id, strategy_id());
19391 assert_eq!(
19392 event.reason.as_str(),
19393 if event_info == "application" {
19394 "Lighter modify transaction failed: invalid order price"
19395 } else {
19396 "Lighter modify transaction failed: transaction failed without an application error"
19397 }
19398 );
19399 assert_eq!(client.dispatch.pending_modify_nonce(&id), None);
19400 assert_eq!(client.dispatch.pending_order_action(&id), None);
19401 assert!(resolve_acked_action(&pending, &context, tx).await.unwrap());
19402 } else {
19403 assert_eq!(client.dispatch.pending_modify_nonce(&id), Some((0, nonce)));
19404 }
19405
19406 assert!(rx.try_recv().is_err());
19407 assert_eq!(
19408 client
19409 .dispatch
19410 .nonce_manager
19411 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
19412 .unwrap(),
19413 nonce + 1
19414 );
19415 }
19416
19417 #[rstest]
19418 #[case::empty_error("")]
19419 #[case::malformed_error("invalid json")]
19420 #[tokio::test]
19421 async fn acknowledged_cancel_failure_consumes_action_once(
19422 #[case] event_info: &str,
19423 #[values(false, true)] emit_rejected: bool,
19424 ) {
19425 let (client, cache, mut rx) = create_execution_client();
19426 let instrument = register_test_instrument(&client, &cache);
19427 let id = ClientOrderId::from("CANCEL-FAILED");
19428 let venue = VenueOrderId::from("789");
19429 let nonce = client
19430 .dispatch
19431 .nonce_manager
19432 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
19433 .unwrap();
19434 enqueue_cancel(&client, instrument, id, venue, nonce);
19435 let mut pending = client.dispatch.pop_pending_sendtx_head().unwrap();
19436
19437 if let PendingSendTxKind::Cancel {
19438 emit_cancel_rejected,
19439 ..
19440 } = &mut pending.kind
19441 {
19442 *emit_cancel_rejected = emit_rejected;
19443 }
19444
19445 client
19446 .dispatch
19447 .set_pending_order_action(id, PendingOrderAction::cancel(emit_rejected));
19448 client
19449 .dispatch
19450 .set_pending_cancel_nonce(&id, 0, nonce, emit_rejected);
19451 let mut tx = failed_modify_tx(&pending);
19452 tx.tx_type = LighterTxType::CancelOrder as u8;
19453 tx.event_info = event_info.to_string();
19454 let context = acked_probe_context(&client);
19455 let resolved = resolve_acked_action(&pending, &context, tx.clone())
19456 .await
19457 .unwrap();
19458
19459 if emit_rejected {
19460 let OrderEventAny::CancelRejected(event) = recv_order_event(&mut rx).await else {
19461 panic!("expected CancelRejected")
19462 };
19463
19464 assert_eq!(event.client_order_id, id);
19465 assert_eq!(event.venue_order_id, Some(venue));
19466 assert_eq!(event.instrument_id, instrument);
19467 assert_eq!(event.strategy_id, strategy_id());
19468 assert_eq!(
19469 event.reason.as_str(),
19470 "Lighter cancel transaction failed: transaction failed without an application error"
19471 );
19472 }
19473
19474 let duplicate_resolved = resolve_acked_action(&pending, &context, tx).await.unwrap();
19475 assert!(resolved);
19476 assert!(duplicate_resolved);
19477 assert_eq!(client.dispatch.pending_cancel_nonce(&id), None);
19478 assert_eq!(client.dispatch.pending_order_action(&id), None);
19479 assert!(rx.try_recv().is_err());
19480 assert_eq!(
19481 client
19482 .dispatch
19483 .nonce_manager
19484 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX,)
19485 .unwrap(),
19486 nonce + 1
19487 );
19488 }
19489
19490 #[rstest]
19491 #[case::newer_nonce(false)]
19492 #[case::reconnected(true)]
19493 #[tokio::test]
19494 async fn acknowledged_modify_does_not_consume_newer_submission(#[case] reconnected: bool) {
19495 let (client, cache, mut rx) = create_execution_client();
19496 let instrument = register_test_instrument(&client, &cache);
19497 let id = ClientOrderId::from("MODIFY-NEWER");
19498 let venue = VenueOrderId::from("789");
19499 enqueue_modify(&client, instrument, id, venue, 23);
19500 let pending = client.dispatch.pop_pending_sendtx_head().unwrap();
19501 let context = acked_probe_context(&client);
19502 client.dispatch.clear_pending_modify_if_nonce(&id, 0, 23);
19503
19504 if reconnected {
19505 context.connection_epoch.store(1, Ordering::Release);
19506 enqueue_modify(&client, instrument, id, venue, 24);
19507 } else {
19508 enqueue_modify(&client, instrument, id, venue, 24);
19509 }
19510
19511 assert!(
19512 resolve_acked_action(&pending, &context, failed_modify_tx(&pending))
19513 .await
19514 .unwrap()
19515 );
19516 assert_eq!(client.dispatch.pending_modify_nonce(&id), Some((0, 24)));
19517 assert!(rx.try_recv().is_err());
19518 }
19519
19520 #[rstest]
19521 fn queued_modify_cannot_bind_a_newer_command() {
19522 let (client, cache, _rx) = create_execution_client();
19523 let instrument = register_test_instrument(&client, &cache);
19524 let id = ClientOrderId::from("MODIFY-QUEUED");
19525
19526 let mut cmd = ModifyOrder::new(
19527 trader_id(),
19528 Some(client_id()),
19529 strategy_id(),
19530 instrument,
19531 id,
19532 Some(VenueOrderId::from("789")),
19533 Some(Quantity::from("0.2000")),
19534 Some(Price::from("2362.00")),
19535 None,
19536 UUID4::new(),
19537 UnixNanos::default(),
19538 None,
19539 None,
19540 );
19541 let old_command = cmd.command_id;
19542 let old_cmd = cmd.clone();
19543 client.dispatch.set_pending_modify_action(&cmd);
19544 assert!(
19545 client
19546 .dispatch
19547 .clear_pending_modify_if_command(&id, old_command)
19548 );
19549 cmd.command_id = UUID4::new();
19550 cmd.price = Some(Price::from("2363.00"));
19551 client.dispatch.set_pending_modify_action(&cmd);
19552 client.dispatch.set_pending_modify_shape(
19553 &cmd,
19554 OrderShapeSnapshot {
19555 quantity: cmd.quantity.unwrap(),
19556 price: cmd.price,
19557 trigger_price: cmd.trigger_price,
19558 },
19559 );
19560
19561 client.dispatch.set_pending_modify_shape(
19562 &old_cmd,
19563 OrderShapeSnapshot {
19564 quantity: old_cmd.quantity.unwrap(),
19565 price: old_cmd.price,
19566 trigger_price: old_cmd.trigger_price,
19567 },
19568 );
19569
19570 client
19571 .dispatch
19572 .set_pending_modify_nonce(&id, old_command, 0, 23);
19573 assert_eq!(client.dispatch.pending_modify_nonce(&id), None);
19574 assert!(
19575 !client
19576 .dispatch
19577 .clear_pending_modify_if_command(&id, old_command)
19578 );
19579 assert_eq!(
19580 client.dispatch.pending_order_action(&id),
19581 Some(PendingOrderAction::Modify)
19582 );
19583 client
19584 .dispatch
19585 .set_pending_modify_nonce(&id, cmd.command_id, 0, 24);
19586 client.dispatch.clear_pending_modify_if_update(
19587 &id,
19588 Quantity::from("0.2000"),
19589 Some(Price::from("2362.00")),
19590 None,
19591 );
19592 assert_eq!(client.dispatch.pending_modify_nonce(&id), Some((0, 24)));
19593 client.dispatch.clear_pending_modify_if_update(
19594 &id,
19595 Quantity::from("0.2000"),
19596 Some(Price::from("2363.00")),
19597 None,
19598 );
19599 assert_eq!(client.dispatch.pending_order_action(&id), None);
19600 }
19601
19602 #[tokio::test]
19603 async fn handle_send_tx_ack_removes_hash_matched_entry() {
19604 let (client, cache, mut rx) = create_execution_client();
19605 let instrument_id = register_test_instrument(&client, &cache);
19606 let mut factory = test_order_factory();
19607 let order_a = test_limit_order(&mut factory, instrument_id, "ACK-A");
19608 let order_b = test_limit_order(&mut factory, instrument_id, "ACK-B");
19609 enqueue_create(&client, &order_a, 10);
19610 enqueue_create(&client, &order_b, 11);
19611
19612 let acked = handle_send_tx_ack(
19614 &client.dispatch,
19615 Some(TEST_ACCOUNT_INDEX_I64),
19616 200,
19617 Some("hash0b"),
19618 );
19619
19620 let acked = acked.expect("create ack");
19621 assert!(matches!(acked.kind, PendingSendTxKind::Create { .. }));
19622 assert!(matches!(
19623 AckedOrderProbe::from_pending(&acked),
19624 Some(AckedOrderProbe {
19625 nonce: 11,
19626 connection_epoch: 0,
19627 ..
19628 }),
19629 ));
19630 assert_eq!(client.dispatch.pending_sendtx_len(), 1, "only B pops");
19631 let head = client.dispatch.pop_pending_sendtx_head().unwrap();
19632 match head.kind {
19633 PendingSendTxKind::Create { order, .. } => {
19634 assert_eq!(order.client_order_id(), order_a.client_order_id());
19635 }
19636 _ => panic!("expected Create kind"),
19637 }
19638
19639 assert!(
19640 tokio::time::timeout(Duration::from_millis(50), rx.recv())
19641 .await
19642 .is_err(),
19643 "ack must not emit an event",
19644 );
19645 }
19646
19647 #[rstest]
19648 #[case::executed(LighterTxStatus::Executed, false)]
19649 #[case::pending_final(LighterTxStatus::PendingFinal, true)]
19650 #[tokio::test]
19651 async fn acknowledged_create_probe_retains_unqueryable_order(
19652 #[case] status: LighterTxStatus,
19653 #[case] pending: bool,
19654 ) {
19655 let (client, cache, mut rx) = create_execution_client();
19656 let instrument_id = register_test_instrument(&client, &cache);
19657 let mut factory = test_order_factory();
19658 let order = test_limit_order(&mut factory, instrument_id, "PROBE-UNQUERYABLE");
19659 let index = enqueue_create(&client, &order, TEST_SUBMISSION_NONCE);
19660 let submission = client.dispatch.pop_pending_sendtx_head().unwrap();
19661
19662 let tx = LighterTx {
19663 code: 200,
19664 message: None,
19665 hash: submission.tx_hash.clone(),
19666 tx_type: LighterTxType::CreateOrder as u8,
19667 info: serde_json::json!({"ClientOrderIndex": index}).to_string(),
19668 event_info: serde_json::json!({"ae": ""}).to_string(),
19669 status,
19670 account_index: TEST_ACCOUNT_INDEX_I64,
19671 nonce: TEST_SUBMISSION_NONCE,
19672 api_key_index: TEST_API_KEY_INDEX,
19673 };
19674
19675 let (http_client, server) = spawn_create_probe_server(&tx).await;
19676 let mut context = acked_probe_context(&client);
19677 context.http_client = http_client;
19678 let cloid = order.client_order_id();
19679 assert!(
19680 client
19681 .dispatch
19682 .create_submission_is_pending(&cloid, index, TEST_SUBMISSION_NONCE)
19683 );
19684
19685 probe_acked_create(
19686 &AckedOrderProbe::from_pending(&submission).unwrap(),
19687 &context,
19688 )
19689 .await
19690 .unwrap();
19691 server.abort();
19692 let _ = server.await;
19693
19694 let identity = client.dispatch.order_identity(&cloid).unwrap();
19695 assert_eq!(
19696 client
19697 .dispatch
19698 .create_submission_is_pending(&cloid, index, TEST_SUBMISSION_NONCE),
19699 pending
19700 );
19701 assert_eq!(identity.instrument_id, instrument_id);
19702 assert_eq!(identity.strategy_id, strategy_id());
19703 assert_eq!(identity.order_side, order.order_side());
19704 assert_eq!(identity.order_type, order.order_type());
19705 assert_eq!(identity.client_order_index, index);
19706 assert_eq!(identity.venue_order_id(), None);
19707 assert!(!client.dispatch.accepted_was_emitted(&cloid));
19708 assert!(!client.dispatch.venue_id_map.contains_key(&cloid));
19709 assert_eq!(
19710 client.dispatch.cloid_map.get(&index).map(|id| *id),
19711 Some(cloid)
19712 );
19713 assert!(drain_events(&mut rx).is_empty());
19714 }
19715
19716 #[rstest]
19717 #[case::hash("hash")]
19718 #[case::type_mismatch("type")]
19719 #[case::account("account")]
19720 #[case::key("key")]
19721 #[case::nonce("nonce")]
19722 #[case::index("index")]
19723 #[tokio::test]
19724 async fn acknowledged_create_probe_requires_exact_http_transaction_identity(
19725 #[case] mismatch: &str,
19726 ) {
19727 let (client, cache, mut rx) = create_execution_client();
19728 let instrument_id = register_test_instrument(&client, &cache);
19729 let mut factory = test_order_factory();
19730 let order = test_limit_order(&mut factory, instrument_id, "PROBE-IDENTITY");
19731 let index = enqueue_create(&client, &order, TEST_SUBMISSION_NONCE);
19732 let submission = client.dispatch.pop_pending_sendtx_head().unwrap();
19733
19734 let mut tx = LighterTx {
19735 code: 200,
19736 message: None,
19737 hash: submission.tx_hash.clone(),
19738 tx_type: LighterTxType::CreateOrder as u8,
19739 info: serde_json::json!({"ClientOrderIndex": index}).to_string(),
19740 event_info: serde_json::json!({"ae": "foreign transaction failed"}).to_string(),
19741 status: LighterTxStatus::Failed,
19742 account_index: TEST_ACCOUNT_INDEX_I64,
19743 nonce: TEST_SUBMISSION_NONCE,
19744 api_key_index: TEST_API_KEY_INDEX,
19745 };
19746
19747 match mismatch {
19748 "hash" => tx.hash = "foreign".to_string(),
19749 "type" => tx.tx_type = LighterTxType::CancelOrder as u8,
19750 "account" => tx.account_index += 1,
19751 "key" => tx.api_key_index += 1,
19752 "nonce" => tx.nonce += 1,
19753 "index" => tx.info = serde_json::json!({"ClientOrderIndex": index + 1}).to_string(),
19754 _ => unreachable!(),
19755 }
19756
19757 let (http_client, server) = spawn_create_probe_server(&tx).await;
19758 let mut context = acked_probe_context(&client);
19759 context.http_client = http_client;
19760 let error = probe_acked_create(
19761 &AckedOrderProbe::from_pending(&submission).unwrap(),
19762 &context,
19763 )
19764 .await
19765 .unwrap_err();
19766 server.abort();
19767 let _ = server.await;
19768
19769 let expected = if mismatch == "index" {
19770 format!(
19771 "Lighter transaction lookup returned client_order_index {} for acknowledged create {index}",
19772 index + 1
19773 )
19774 } else {
19775 "Lighter transaction lookup did not match acknowledged create identity".to_string()
19776 };
19777
19778 let cloid = order.client_order_id();
19779 assert_eq!(error.to_string(), expected);
19780 assert!(
19781 client
19782 .dispatch
19783 .create_submission_is_pending(&cloid, index, TEST_SUBMISSION_NONCE)
19784 );
19785 assert_eq!(
19786 client
19787 .dispatch
19788 .order_identity(&cloid)
19789 .unwrap()
19790 .client_order_index,
19791 index
19792 );
19793 assert_eq!(
19794 client.dispatch.cloid_map.get(&index).map(|id| *id),
19795 Some(cloid)
19796 );
19797 assert!(!client.dispatch.accepted_was_emitted(&cloid));
19798 assert!(!client.dispatch.venue_id_map.contains_key(&cloid));
19799 assert!(drain_events(&mut rx).is_empty());
19800 }
19801
19802 async fn spawn_create_probe_server(
19803 tx: &LighterTx,
19804 ) -> (LighterHttpClient, tokio::task::JoinHandle<()>) {
19805 let body = serde_json::to_string(tx).unwrap();
19806
19807 let app = Router::new()
19808 .route(
19809 "/api/v1/accountActiveOrders",
19810 get(|| async { r#"{"code":200,"orders":[]}"# }),
19811 )
19812 .route(
19813 "/api/v1/tx",
19814 get(move || {
19815 let body = body.clone();
19816 async move { body }
19817 }),
19818 );
19819
19820 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
19821 let addr = listener.local_addr().unwrap();
19822 let server = tokio::spawn(async move {
19823 axum::serve(listener, app).await.unwrap();
19824 });
19825
19826 let client = LighterHttpClient::new(
19827 LighterEnvironment::Testnet,
19828 Some(format!("http://{addr}")),
19829 1,
19830 None,
19831 )
19832 .unwrap();
19833 (client, server)
19834 }
19835
19836 #[rstest]
19837 fn acknowledged_create_tx_validation_requires_exact_identity() {
19838 let tx = LighterTx {
19839 code: 200,
19840 message: None,
19841 hash: "ABCDEF".to_string(),
19842 tx_type: LighterTxType::CreateOrder as u8,
19843 info: serde_json::json!({"ClientOrderIndex": 42}).to_string(),
19844 event_info: serde_json::json!({"ae": ""}).to_string(),
19845 status: LighterTxStatus::Failed,
19846 account_index: TEST_ACCOUNT_INDEX_I64,
19847 nonce: 10,
19848 api_key_index: TEST_API_KEY_INDEX,
19849 };
19850 assert!(
19851 validate_acked_create_tx(
19852 &tx,
19853 TEST_ACCOUNT_INDEX_I64,
19854 TEST_API_KEY_INDEX,
19855 42,
19856 10,
19857 "0xabcdef",
19858 )
19859 .is_ok(),
19860 );
19861
19862 let mismatches = [
19863 LighterTx {
19864 hash: "different".to_string(),
19865 ..tx.clone()
19866 },
19867 LighterTx {
19868 tx_type: LighterTxType::CancelOrder as u8,
19869 ..tx.clone()
19870 },
19871 LighterTx {
19872 account_index: TEST_ACCOUNT_INDEX_I64 + 1,
19873 ..tx.clone()
19874 },
19875 LighterTx {
19876 nonce: 11,
19877 ..tx.clone()
19878 },
19879 LighterTx {
19880 api_key_index: TEST_API_KEY_INDEX + 1,
19881 ..tx.clone()
19882 },
19883 ];
19884
19885 for mismatch in mismatches {
19886 let error = validate_acked_create_tx(
19887 &mismatch,
19888 TEST_ACCOUNT_INDEX_I64,
19889 TEST_API_KEY_INDEX,
19890 42,
19891 10,
19892 "abcdef",
19893 )
19894 .expect_err("mismatched transaction identity must fail");
19895 assert_eq!(
19896 error.to_string(),
19897 "Lighter transaction lookup did not match acknowledged create identity",
19898 );
19899 }
19900
19901 let wrong_index = LighterTx {
19902 info: serde_json::json!({"ClientOrderIndex": 43}).to_string(),
19903 ..tx
19904 };
19905
19906 let error = validate_acked_create_tx(
19907 &wrong_index,
19908 TEST_ACCOUNT_INDEX_I64,
19909 TEST_API_KEY_INDEX,
19910 42,
19911 10,
19912 "abcdef",
19913 )
19914 .expect_err("mismatched client order index must fail");
19915 assert_eq!(
19916 error.to_string(),
19917 "Lighter transaction lookup returned client_order_index 43 for acknowledged create 42",
19918 );
19919 }
19920
19921 #[rstest]
19922 #[case(41, true)]
19923 #[case(42, true)]
19924 #[case(43, true)]
19925 #[case(44, false)]
19926 fn acknowledged_group_tx_validation_requires_membership(
19927 #[case] client_order_index: i64,
19928 #[case] expected: bool,
19929 ) {
19930 let tx = LighterTx {
19931 code: 200,
19932 message: None,
19933 hash: "ABCDEF".to_string(),
19934 tx_type: LighterTxType::CreateGroupedOrders as u8,
19935 info: serde_json::json!({"Orders": [
19936 {"ClientOrderIndex": 41}, {"ClientOrderIndex": 42}, {"ClientOrderIndex": 43}
19937 ]})
19938 .to_string(),
19939 event_info: serde_json::json!({"ae": ""}).to_string(),
19940 status: LighterTxStatus::Executed,
19941 account_index: TEST_ACCOUNT_INDEX_I64,
19942 nonce: 10,
19943 api_key_index: TEST_API_KEY_INDEX,
19944 };
19945
19946 let result = validate_acked_create_tx(
19947 &tx,
19948 TEST_ACCOUNT_INDEX_I64,
19949 TEST_API_KEY_INDEX,
19950 client_order_index,
19951 10,
19952 "0xabcdef",
19953 );
19954 assert_eq!(result.is_ok(), expected);
19955
19956 if !expected {
19957 assert_eq!(
19958 result.unwrap_err().to_string(),
19959 "Lighter transaction lookup did not include acknowledged client_order_index 44"
19960 );
19961 }
19962 }
19963
19964 #[tokio::test]
19965 async fn send_tx_responses_only_attribute_within_their_connection_epoch() {
19966 let (client, _cache, mut rx) = create_execution_client();
19967 for (connection_epoch, nonce) in [(4, 20), (5, 21)] {
19968 client.dispatch.enqueue_pending_sendtx(PendingSendTx {
19969 batch_id: None,
19970 connection_epoch,
19971 kind: PendingSendTxKind::Other,
19972 submitted_at: UnixNanos::from(1_000_000_000 + nonce as u64),
19973 nonce,
19974 api_key_index: TEST_API_KEY_INDEX,
19975 tx_hash: "same-hash".to_string(),
19976 });
19977 }
19978
19979 let acked =
19980 handle_send_tx_ack_for_connection(&client.dispatch, None, 5, 200, Some("same-hash"))
19981 .expect("replacement epoch response should match");
19982 assert_eq!(acked.connection_epoch, 5);
19983 assert_eq!(acked.nonce, 21);
19984 {
19985 let queue = client.dispatch.pending_sendtx.lock();
19986 assert_eq!(queue.len(), 1);
19987 assert_eq!(queue[0].connection_epoch, 4);
19988 assert_eq!(queue[0].nonce, 20);
19989 }
19990
19991 assert!(!handle_send_tx_rejection_for_connection(
19992 &client.dispatch,
19993 &client.emitter,
19994 None,
19995 4,
19996 UnixNanos::from(1_000_000_100),
19997 SendTxRejectionSource::Ack,
19998 Some(21702),
19999 "invalid price",
20000 Some("same-hash"),
20001 ));
20002 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
20003 assert!(
20004 tokio::time::timeout(Duration::from_millis(50), rx.recv())
20005 .await
20006 .is_err(),
20007 "non-order responses must not emit order events",
20008 );
20009 }
20010
20011 #[tokio::test]
20012 async fn handle_send_tx_ack_unmatched_hash_pops_nothing() {
20013 let (client, cache, mut rx) = create_execution_client();
20014 let instrument_id = register_test_instrument(&client, &cache);
20015 let mut factory = test_order_factory();
20016 let order = test_limit_order(&mut factory, instrument_id, "ACK-UNMATCHED");
20017 enqueue_create(&client, &order, 10);
20018
20019 let acked = handle_send_tx_ack(
20020 &client.dispatch,
20021 Some(TEST_ACCOUNT_INDEX_I64),
20022 200,
20023 Some("0xabc"),
20024 );
20025
20026 assert!(acked.is_none());
20027 assert_eq!(
20028 client.dispatch.pending_sendtx_len(),
20029 1,
20030 "an echoed hash with no matching entry must not pop the head",
20031 );
20032 assert!(
20033 tokio::time::timeout(Duration::from_millis(50), rx.recv())
20034 .await
20035 .is_err(),
20036 "unmatched ack must not emit an event",
20037 );
20038 }
20039
20040 #[tokio::test]
20041 async fn handle_send_tx_ack_hashless_with_multiple_pending_pops_nothing() {
20042 let (client, _cache, _rx) = create_execution_client();
20043 enqueue_other(&client, 10);
20044 enqueue_other(&client, 11);
20045
20046 let acked = handle_send_tx_ack(&client.dispatch, Some(TEST_ACCOUNT_INDEX_I64), 200, None);
20047
20048 assert!(acked.is_none());
20049 assert_eq!(
20050 client.dispatch.pending_sendtx_len(),
20051 2,
20052 "a hashless ack is ambiguous when multiple transactions are pending",
20053 );
20054 }
20055
20056 #[tokio::test]
20057 async fn prepared_create_tx_hash_round_trips_through_ack() {
20058 let (client, cache, _rx) = create_execution_client();
20061 let instrument_id = register_test_instrument(&client, &cache);
20062 let mut factory = test_order_factory();
20063 let order = test_limit_order(&mut factory, instrument_id, "ACK-REAL-HASH");
20064
20065 let credential = test_credential();
20066 let plan = client
20067 .prepare_create_order_plan(&order, 0, None)
20068 .expect("prepare must validate");
20069 let prepared = client
20070 .fanout_dispatch_context(&credential)
20071 .expect("task admission")
20072 .sign_create_order(&plan)
20073 .expect("prepare must sign");
20074
20075 assert_eq!(prepared.tx_hash.len(), TX_HASH_BYTES * 2);
20076 assert!(
20077 prepared
20078 .tx_hash
20079 .chars()
20080 .all(|c| matches!(c, '0'..='9' | 'a'..='f')),
20081 "tx_hash must be lowercase hex, was `{}`",
20082 prepared.tx_hash,
20083 );
20084
20085 client.dispatch.enqueue_pending_sendtx(PendingSendTx {
20086 batch_id: None,
20087 connection_epoch: 0,
20088 kind: PendingSendTxKind::Create {
20089 order: Box::new(order),
20090 client_order_index: prepared.client_order_index,
20091 },
20092 submitted_at: UnixNanos::from(1_000_000_000),
20093 nonce: prepared.nonce,
20094 api_key_index: prepared.api_key_index,
20095 tx_hash: prepared.tx_hash.clone(),
20096 });
20097
20098 let acked = handle_send_tx_ack(
20099 &client.dispatch,
20100 Some(TEST_ACCOUNT_INDEX_I64),
20101 200,
20102 Some(&prepared.tx_hash),
20103 );
20104
20105 assert!(acked.is_some());
20106 assert_eq!(
20107 client.dispatch.pending_sendtx_len(),
20108 0,
20109 "venue echo of the signed hash must match the enqueued entry",
20110 );
20111 }
20112
20113 #[tokio::test]
20114 async fn handle_send_tx_ack_retains_cancel_for_transaction_probe() {
20115 let (client, cache, mut rx) = create_execution_client();
20116 let instrument_id = register_test_instrument(&client, &cache);
20117 let client_order_id = ClientOrderId::from("ACK-CANCEL-TX");
20118 let venue_order_id = VenueOrderId::from("123");
20119 enqueue_cancel(&client, instrument_id, client_order_id, venue_order_id, 12);
20120
20121 let acked = handle_send_tx_ack(
20122 &client.dispatch,
20123 Some(TEST_ACCOUNT_INDEX_I64),
20124 200,
20125 Some("hash0c"),
20126 )
20127 .expect("acked cancel pending entry");
20128 assert!(AckedOrderProbe::from_pending(&acked).is_none());
20129
20130 let PendingSendTxKind::Cancel {
20131 instrument_id: actual_instrument_id,
20132 client_order_id: actual_client_order_id,
20133 venue_order_id: actual_venue_order_id,
20134 emit_cancel_rejected,
20135 strategy_id: actual_strategy_id,
20136 } = acked.kind
20137 else {
20138 panic!("expected acknowledged cancel")
20139 };
20140
20141 assert_eq!(actual_instrument_id, instrument_id);
20142 assert_eq!(actual_client_order_id, client_order_id);
20143 assert_eq!(actual_venue_order_id, Some(venue_order_id));
20144 assert_eq!(actual_strategy_id, strategy_id());
20145 assert!(emit_cancel_rejected);
20146 assert_eq!(acked.connection_epoch, 0);
20147 assert_eq!(acked.nonce, 12);
20148 assert_eq!(acked.api_key_index, TEST_API_KEY_INDEX);
20149 assert_eq!(acked.tx_hash, "hash0c");
20150 assert_eq!(
20151 client.dispatch.pending_cancel_nonce(&client_order_id),
20152 Some((0, 12))
20153 );
20154 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
20155 assert!(
20156 tokio::time::timeout(Duration::from_millis(50), rx.recv())
20157 .await
20158 .is_err(),
20159 "ack itself must not emit before the transaction probe runs",
20160 );
20161 }
20162
20163 #[rstest]
20164 #[tokio::test]
20165 async fn acknowledged_order_probe_is_retained_until_session_shutdown() {
20166 let (mut client, cache, _rx) = create_execution_client();
20167 let instrument_id = register_test_instrument(&client, &cache);
20168 let pending = PendingSendTx {
20169 batch_id: None,
20170 connection_epoch: 0,
20171 kind: PendingSendTxKind::Cancel {
20172 emit_cancel_rejected: true,
20173 strategy_id: strategy_id(),
20174 instrument_id,
20175 client_order_id: ClientOrderId::from("ACK-RETAINED"),
20176 venue_order_id: Some(VenueOrderId::from("123")),
20177 },
20178 submitted_at: UnixNanos::from(1_000_000_000),
20179 nonce: 12,
20180 api_key_index: TEST_API_KEY_INDEX,
20181 tx_hash: "hash0c".to_string(),
20182 };
20183 let context = AckedOrderProbeContext {
20184 http_client: client.http_client.clone(),
20185 registry: Arc::clone(&client.registry),
20186 credential: test_credential(),
20187 dispatch: client.dispatch.clone(),
20188 account_id: client.core.account_id,
20189 clock: client.clock,
20190 emitter: client.emitter.clone(),
20191 connection_epoch: client.ws_client.connection_epoch_atomic(),
20192 cancellation_token: client.cancellation_token.clone(),
20193 pending_tasks: client.pending_tasks.spawner().expect("task spawner"),
20194 };
20195
20196 spawn_acked_order_probe(&pending, context);
20197
20198 assert_eq!(client.pending_tasks.len(), 1);
20199
20200 client.begin_session_shutdown();
20201 client
20202 .finish_session_shutdown()
20203 .await
20204 .expect("session shutdown");
20205
20206 assert!(client.pending_tasks.is_empty());
20207 }
20208
20209 #[tokio::test]
20210 async fn handle_send_tx_ack_retains_modify_for_transaction_probe() {
20211 let (client, cache, mut rx) = create_execution_client();
20212 let instrument_id = register_test_instrument(&client, &cache);
20213 let client_order_id = ClientOrderId::from("ACK-MODIFY-TX");
20214 let venue_order_id = VenueOrderId::from("456");
20215 enqueue_modify(&client, instrument_id, client_order_id, venue_order_id, 13);
20216
20217 let acked = handle_send_tx_ack(
20218 &client.dispatch,
20219 Some(TEST_ACCOUNT_INDEX_I64),
20220 200,
20221 Some("hash0d"),
20222 )
20223 .expect("acked modify pending entry");
20224 assert!(AckedOrderProbe::from_pending(&acked).is_none());
20225
20226 match acked.kind {
20227 PendingSendTxKind::Modify {
20228 instrument_id: actual_instrument_id,
20229 client_order_id: actual_client_order_id,
20230 venue_order_id: actual_venue_order_id,
20231 ..
20232 } => {
20233 assert_eq!(actual_instrument_id, instrument_id);
20234 assert_eq!(actual_client_order_id, client_order_id);
20235 assert_eq!(actual_venue_order_id, Some(venue_order_id));
20236 }
20237 other => panic!("expected acknowledged modify, was {other:?}"),
20238 }
20239
20240 assert_eq!(acked.connection_epoch, 0);
20241 assert_eq!(acked.nonce, 13);
20242 assert_eq!(acked.api_key_index, TEST_API_KEY_INDEX);
20243 assert_eq!(acked.tx_hash, "hash0d");
20244 assert_eq!(
20245 client.dispatch.pending_modify_nonce(&client_order_id),
20246 Some((0, 13))
20247 );
20248 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
20249 assert!(
20250 tokio::time::timeout(Duration::from_millis(50), rx.recv())
20251 .await
20252 .is_err(),
20253 "ack itself must not emit before the transaction probe runs",
20254 );
20255 }
20256
20257 #[rstest]
20258 #[case::missing(false)]
20259 #[case::working(true)]
20260 #[tokio::test]
20261 async fn acknowledged_cancel_unresolved_status_retains_pending(#[case] found: bool) {
20262 let (client, cache, mut rx) = create_execution_client();
20263 let instrument = register_test_instrument(&client, &cache);
20264 let id = ClientOrderId::from("ACK-CANCEL-UNRESOLVED");
20265 let venue = VenueOrderId::from("123");
20266 let mut factory = test_order_factory();
20267 let order = test_limit_order(&mut factory, instrument, id.as_str());
20268 let index = client.dispatch.register_create_identity(&order).unwrap();
20269 enqueue_cancel(&client, instrument, id, venue, 12);
20270 let pending = client.dispatch.pop_pending_sendtx_head().unwrap();
20271 let order = reconciliation_raw_order(index, venue, LighterOrderStatus::Open);
20272
20273 let orders = if found { vec![order] } else { vec![] };
20274 let app = Router::new()
20275 .route(
20276 "/api/v1/accountActiveOrders",
20277 get(move || {
20278 let orders = orders.clone();
20279 async move { axum::Json(serde_json::json!({"code":200,"orders":orders})) }
20280 }),
20281 )
20282 .route(
20283 "/api/v1/accountInactiveOrders",
20284 get(|| async { axum::Json(serde_json::json!({"code":200,"orders":[]})) }),
20285 );
20286
20287 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
20288 let addr = listener.local_addr().unwrap();
20289 let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
20290 let mut context = acked_probe_context(&client);
20291 context.http_client = LighterHttpClient::new(
20292 LighterEnvironment::Testnet,
20293 Some(format!("http://{addr}")),
20294 1,
20295 None,
20296 )
20297 .unwrap();
20298 let resolved = recover_acked_action_report(&pending, &context)
20299 .await
20300 .unwrap();
20301 server.abort();
20302 let _ = server.await;
20303 assert!(!resolved);
20304 assert_eq!(client.dispatch.pending_cancel_nonce(&id), Some((0, 12)));
20305 assert_eq!(
20306 client.dispatch.pending_order_action(&id),
20307 Some(PendingOrderAction::Cancel)
20308 );
20309 let events = drain_events(&mut rx);
20310 if found {
20311 let [ExecutionEvent::Report(ExecutionReport::Order(report))] = events.as_slice() else {
20312 panic!("expected pending cancellation report, was {events:?}")
20313 };
20314
20315 assert_eq!(report.client_order_id, Some(id));
20316 assert_eq!(report.venue_order_id, venue);
20317 assert_eq!(report.order_status, OrderStatus::PendingCancel);
20318 } else {
20319 assert!(events.is_empty());
20320 }
20321 }
20322
20323 #[tokio::test]
20324 async fn send_tx_acks_recover_skip_window_past_window_size() {
20325 let (client, _cache, _rx) = create_execution_client();
20326 let window = i64::from(client.dispatch.nonce_manager.skip_window());
20327 for _ in 0..window {
20328 let nonce = client
20329 .dispatch
20330 .nonce_manager
20331 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
20332 .unwrap();
20333 enqueue_other(&client, nonce);
20334 }
20335
20336 assert!(
20337 client
20338 .dispatch
20339 .nonce_manager
20340 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
20341 .is_err(),
20342 "window must trip after {window} unacked txs",
20343 );
20344
20345 for i in 0..window {
20348 let acked = handle_send_tx_ack(
20349 &client.dispatch,
20350 Some(TEST_ACCOUNT_INDEX_I64),
20351 200,
20352 Some(&format!("hash{:02x}", TEST_NEXT_NONCE + i)),
20353 );
20354 assert!(matches!(
20355 acked.map(|pending| pending.kind),
20356 Some(PendingSendTxKind::Other),
20357 ));
20358 let nonce = client
20359 .dispatch
20360 .nonce_manager
20361 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
20362 .unwrap();
20363 assert_eq!(nonce, TEST_NEXT_NONCE + window + i);
20364 enqueue_other(&client, nonce);
20365 }
20366 }
20367
20368 #[tokio::test]
20369 async fn handle_send_tx_rejection_rolls_back_latest_nonce() {
20370 let (client, cache, mut rx) = create_execution_client();
20371 let instrument_id = register_test_instrument(&client, &cache);
20372 let mut factory = test_order_factory();
20373 let order = test_limit_order(&mut factory, instrument_id, "REJECT-LATEST");
20374
20375 let nonce = client
20376 .dispatch
20377 .nonce_manager
20378 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
20379 .unwrap();
20380 let client_order_index = client.dispatch.register_create_identity(&order).unwrap();
20381 client.dispatch.enqueue_pending_sendtx(PendingSendTx {
20382 batch_id: None,
20383 connection_epoch: 0,
20384 kind: PendingSendTxKind::Create {
20385 order: Box::new(order.clone()),
20386 client_order_index,
20387 },
20388 submitted_at: UnixNanos::from(1_000_000_000),
20389 nonce,
20390 api_key_index: TEST_API_KEY_INDEX,
20391 tx_hash: format!("hash{nonce:02x}"),
20392 });
20393
20394 handle_send_tx_rejection(
20395 &client.dispatch,
20396 &client.emitter,
20397 Some(TEST_ACCOUNT_INDEX_I64),
20398 UnixNanos::from(1_000_000_000),
20399 SendTxRejectionSource::Ack,
20400 Some(21702),
20401 "invalid price",
20402 None,
20403 );
20404
20405 let event = recv_order_event(&mut rx).await;
20406 assert!(matches!(event, OrderEventAny::Rejected(_)));
20407 assert_nonce_reusable(&client.dispatch);
20408 }
20409
20410 #[tokio::test]
20411 async fn handle_send_tx_rejection_hashless_with_multiple_pending_is_unattributed() {
20412 let (client, cache, mut rx) = create_execution_client();
20413 let instrument_id = register_test_instrument(&client, &cache);
20414 let mut factory = test_order_factory();
20415 let order_a = test_limit_order(&mut factory, instrument_id, "REJECT-AMBIGUOUS-A");
20416 let order_b = test_limit_order(&mut factory, instrument_id, "REJECT-AMBIGUOUS-B");
20417 enqueue_create(&client, &order_a, 10);
20418 enqueue_create(&client, &order_b, 11);
20419
20420 handle_send_tx_rejection(
20421 &client.dispatch,
20422 &client.emitter,
20423 Some(TEST_ACCOUNT_INDEX_I64),
20424 UnixNanos::from(1_000_000_000),
20425 SendTxRejectionSource::Ack,
20426 Some(21702),
20427 "invalid price",
20428 None,
20429 );
20430
20431 assert_eq!(client.dispatch.pending_sendtx_len(), 2);
20432 assert!(
20433 tokio::time::timeout(Duration::from_millis(50), rx.recv())
20434 .await
20435 .is_err(),
20436 "an ambiguous hashless rejection must not reject either order",
20437 );
20438 }
20439
20440 #[tokio::test]
20441 async fn handle_send_tx_rejection_with_newer_issuance_skips_rollback() {
20442 let (client, cache, mut rx) = create_execution_client();
20443 let instrument_id = register_test_instrument(&client, &cache);
20444 let mut factory = test_order_factory();
20445 let order = test_limit_order(&mut factory, instrument_id, "REJECT-STALE");
20446
20447 let rejected_nonce = client
20448 .dispatch
20449 .nonce_manager
20450 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
20451 .unwrap();
20452 let newer_nonce = client
20453 .dispatch
20454 .nonce_manager
20455 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
20456 .unwrap();
20457 let client_order_index = client.dispatch.register_create_identity(&order).unwrap();
20458 client.dispatch.enqueue_pending_sendtx(PendingSendTx {
20459 batch_id: None,
20460 connection_epoch: 0,
20461 kind: PendingSendTxKind::Create {
20462 order: Box::new(order.clone()),
20463 client_order_index,
20464 },
20465 submitted_at: UnixNanos::from(1_000_000_000),
20466 nonce: rejected_nonce,
20467 api_key_index: TEST_API_KEY_INDEX,
20468 tx_hash: format!("hash{rejected_nonce:02x}"),
20469 });
20470
20471 handle_send_tx_rejection(
20472 &client.dispatch,
20473 &client.emitter,
20474 Some(TEST_ACCOUNT_INDEX_I64),
20475 UnixNanos::from(1_000_000_000),
20476 SendTxRejectionSource::Ack,
20477 Some(21702),
20478 "invalid price",
20479 None,
20480 );
20481
20482 let event = recv_order_event(&mut rx).await;
20483 assert!(matches!(event, OrderEventAny::Rejected(_)));
20484 assert_eq!(
20487 client
20488 .dispatch
20489 .nonce_manager
20490 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
20491 Some(newer_nonce),
20492 );
20493 assert_eq!(
20494 client
20495 .dispatch
20496 .nonce_manager
20497 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
20498 .unwrap(),
20499 newer_nonce + 1,
20500 );
20501 }
20502
20503 async fn spawn_next_nonce_server(nonce: i64) -> String {
20504 let body = serde_json::to_string(&LighterNextNonce {
20505 code: 200,
20506 message: None,
20507 nonce,
20508 })
20509 .unwrap();
20510
20511 let app = Router::new().route(
20512 "/api/v1/nextNonce",
20513 get(move || {
20514 let body = body.clone();
20515 async move { body }
20516 }),
20517 );
20518
20519 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
20520 let addr = listener.local_addr().unwrap();
20521 tokio::spawn(async move {
20522 axum::serve(listener, app).await.unwrap();
20523 });
20524
20525 format!("http://{addr}")
20526 }
20527
20528 async fn spawn_counting_next_nonce_server(nonce: i64) -> (String, Arc<AtomicUsize>) {
20529 let requests = Arc::new(AtomicUsize::new(0));
20530 let requests_for_route = Arc::clone(&requests);
20531
20532 let body = serde_json::to_string(&LighterNextNonce {
20533 code: 200,
20534 message: None,
20535 nonce,
20536 })
20537 .unwrap();
20538
20539 let app = Router::new().route(
20540 "/api/v1/nextNonce",
20541 get(move || {
20542 requests_for_route.fetch_add(1, Ordering::SeqCst);
20543 let body = body.clone();
20544 async move { body }
20545 }),
20546 );
20547
20548 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
20549 let addr = listener.local_addr().unwrap();
20550 tokio::spawn(async move {
20551 axum::serve(listener, app).await.unwrap();
20552 });
20553
20554 (format!("http://{addr}"), requests)
20555 }
20556
20557 #[tokio::test]
20558 async fn hard_nonce_refresh_waits_for_submission_and_replaces_its_baseline() {
20559 let (base_url_http, requests) = spawn_counting_next_nonce_server(100).await;
20560 let mut config = test_config();
20561 config.base_url_http = Some(base_url_http);
20562 let (client, _cache, _rx) = create_execution_client_with_config(config);
20563 let credential = test_credential();
20564 client
20565 .dispatch
20566 .nonce_manager
20567 .refresh(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX, 40);
20568
20569 let reserved = client.build_tx_context(&credential).unwrap();
20570 assert_eq!(reserved.context.nonce, 40);
20571 let mut refresh = Box::pin(client.refresh_nonce());
20572 assert!(
20573 tokio::time::timeout(Duration::from_millis(50), refresh.as_mut())
20574 .await
20575 .is_err(),
20576 "hard refresh must wait while a submission retains its nonce",
20577 );
20578 assert_eq!(requests.load(Ordering::SeqCst), 0);
20579 assert_eq!(
20580 client.nonce_ready_connection_epoch.load(Ordering::Acquire),
20581 NONCE_CONNECTION_EPOCH_UNAVAILABLE,
20582 );
20583 assert_eq!(
20584 client
20585 .dispatch
20586 .nonce_manager
20587 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
20588 Some(40),
20589 );
20590
20591 drop(reserved);
20592 refresh.await.unwrap();
20593
20594 assert_eq!(requests.load(Ordering::SeqCst), 1);
20595 assert_eq!(
20596 client.nonce_ready_connection_epoch.load(Ordering::Acquire),
20597 NONCE_CONNECTION_EPOCH_UNAVAILABLE,
20598 );
20599 client
20600 .nonce_ready_connection_epoch
20601 .store(client.ws_client.connection_epoch(), Ordering::Release);
20602 let next = client.build_tx_context(&credential).unwrap();
20603 assert_eq!(next.context.nonce, 100);
20604 assert_eq!(
20605 client
20606 .dispatch
20607 .nonce_manager
20608 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
20609 Some(100),
20610 );
20611 }
20612
20613 #[tokio::test]
20614 async fn nonce_refresh_retry_reopens_submission_with_replacement_baseline() {
20615 let (base_url_http, requests) = spawn_counting_next_nonce_server(100).await;
20616 let mut config = test_config();
20617 config.base_url_http = Some(base_url_http);
20618 let (client, _cache, _rx) = create_execution_client_with_config(config);
20619 let credential = test_credential();
20620 let connection_epoch = client.ws_client.connection_epoch();
20621 client
20622 .dispatch
20623 .nonce_manager
20624 .refresh(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX, 40);
20625 client
20626 .nonce_ready_connection_epoch
20627 .store(NONCE_CONNECTION_EPOCH_UNAVAILABLE, Ordering::Release);
20628
20629 NonceRefreshRetry {
20630 http_client: client.http_client.clone(),
20631 dispatch: client.dispatch.clone(),
20632 credential: Some(credential.clone()),
20633 submission_gate: Arc::clone(&client.nonce_submission_gate),
20634 ready_connection_epoch: Arc::clone(&client.nonce_ready_connection_epoch),
20635 ws_client: client.ws_client.clone(),
20636 cancellation_token: client.cancellation_token.clone(),
20637 pending_tasks: client.pending_tasks.spawner().expect("task spawner"),
20638 }
20639 .spawn(connection_epoch);
20640
20641 assert_eq!(client.pending_tasks.len(), 1);
20642
20643 let err = client
20644 .build_tx_context(&credential)
20645 .expect_err("submission must stay closed until retry succeeds");
20646 assert_eq!(
20647 err.to_string(),
20648 format!("Lighter nonce refresh is pending for connection epoch {connection_epoch}"),
20649 );
20650 wait_until_async(
20651 || async {
20652 client.nonce_ready_connection_epoch.load(Ordering::Acquire) == connection_epoch
20653 },
20654 Duration::from_secs(3),
20655 )
20656 .await;
20657
20658 let reserved = client.build_tx_context(&credential).unwrap();
20659 assert_eq!(requests.load(Ordering::SeqCst), 1);
20660 assert_eq!(reserved.context.nonce, 100);
20661 assert_eq!(
20662 client
20663 .dispatch
20664 .nonce_manager
20665 .baseline(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
20666 Some(99),
20667 );
20668 assert_eq!(
20669 client
20670 .dispatch
20671 .nonce_manager
20672 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
20673 Some(100),
20674 );
20675 assert!(client.pending_tasks.all_finished());
20676 }
20677
20678 async fn spawn_integrator_approval_rejection_server() -> String {
20679 let app = Router::new()
20680 .route(
20681 "/api/v1/getMakerOnlyApiKeys",
20682 get(|| async { r#"{"code":200,"api_key_indexes":[]}"# }),
20683 )
20684 .route(
20685 "/api/v1/sendTx",
20686 post(|| async { r#"{"code":21702,"message":"invalid approval"}"# }),
20687 );
20688 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
20689 let addr = listener.local_addr().unwrap();
20690 tokio::spawn(async move {
20691 axum::serve(listener, app).await.unwrap();
20692 });
20693
20694 format!("http://{addr}")
20695 }
20696
20697 #[tokio::test]
20698 async fn integrator_approval_api_rejection_releases_reservation_and_nonce() {
20699 let mut config = test_config();
20700 config.environment = LighterEnvironment::Mainnet;
20701 config.base_url_http = Some(spawn_integrator_approval_rejection_server().await);
20702 let (client, _cache, _rx) = create_execution_client_with_config(config);
20703
20704 let err = client.submit_integrator_auto_approval().await.unwrap_err();
20705
20706 assert!(format!("{err:#}").contains("invalid approval"));
20707 assert_nonce_reusable(&client.dispatch);
20708 }
20709
20710 #[tokio::test]
20711 async fn skip_window_exhaustion_resyncs_baseline_from_venue() {
20712 let venue_next_nonce = 100;
20713 let mut config = test_config();
20714 config.base_url_http = Some(spawn_next_nonce_server(venue_next_nonce).await);
20715 let (client, _cache, _rx) = create_execution_client_with_config(config);
20716 let credential = test_credential();
20717
20718 let window = i64::from(client.dispatch.nonce_manager.skip_window());
20719 for _ in 0..window {
20720 client
20721 .dispatch
20722 .nonce_manager
20723 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
20724 .unwrap();
20725 }
20726
20727 let err = client
20728 .build_tx_context(&credential)
20729 .expect_err("window must trip");
20730 assert!(
20731 err.to_string().contains("skip-window exhausted"),
20732 "unexpected error, was {err}",
20733 );
20734
20735 wait_for_spawned_tasks(&client).await;
20736
20737 assert_eq!(
20738 client
20739 .dispatch
20740 .nonce_manager
20741 .baseline(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
20742 Some(venue_next_nonce - 1),
20743 "venue resync must advance the baseline",
20744 );
20745 assert!(
20746 !client.nonce_recovery_inflight.load(Ordering::Acquire),
20747 "recovery latch must release for the next exhaustion",
20748 );
20749 let context = client.build_tx_context(&credential).unwrap().context;
20750 assert_eq!(
20751 context.nonce, venue_next_nonce,
20752 "allocation must resume at the venue nonce",
20753 );
20754 }
20755
20756 #[tokio::test]
20757 async fn skip_window_recovery_fetch_failure_releases_latch() {
20758 let app = Router::new();
20760 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
20761 let addr = listener.local_addr().unwrap();
20762 tokio::spawn(async move {
20763 axum::serve(listener, app).await.unwrap();
20764 });
20765
20766 let mut config = test_config();
20767 config.base_url_http = Some(format!("http://{addr}"));
20768 let (client, _cache, _rx) = create_execution_client_with_config(config);
20769 let credential = test_credential();
20770
20771 let window = i64::from(client.dispatch.nonce_manager.skip_window());
20772 for _ in 0..window {
20773 client
20774 .dispatch
20775 .nonce_manager
20776 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
20777 .unwrap();
20778 }
20779
20780 client
20781 .build_tx_context(&credential)
20782 .expect_err("window must trip");
20783
20784 wait_for_spawned_tasks(&client).await;
20785
20786 assert_eq!(
20787 client
20788 .dispatch
20789 .nonce_manager
20790 .baseline(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
20791 Some(TEST_NEXT_NONCE - 1),
20792 "failed venue fetch must leave the baseline unchanged",
20793 );
20794 assert!(
20795 !client.nonce_recovery_inflight.load(Ordering::Acquire),
20796 "failed venue fetch must release the recovery latch",
20797 );
20798 }
20799
20800 #[tokio::test]
20801 async fn skip_window_recovery_dedupes_concurrent_fetches() {
20802 let hits = Arc::new(AtomicUsize::new(0));
20803 let server_hits = Arc::clone(&hits);
20804
20805 let body = serde_json::to_string(&LighterNextNonce {
20806 code: 200,
20807 message: None,
20808 nonce: 100,
20809 })
20810 .unwrap();
20811
20812 let app = Router::new().route(
20813 "/api/v1/nextNonce",
20814 get(move || {
20815 server_hits.fetch_add(1, Ordering::AcqRel);
20816 let body = body.clone();
20817 async move { body }
20818 }),
20819 );
20820 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
20821 let addr = listener.local_addr().unwrap();
20822 tokio::spawn(async move {
20823 axum::serve(listener, app).await.unwrap();
20824 });
20825
20826 let mut config = test_config();
20827 config.base_url_http = Some(format!("http://{addr}"));
20828 let (client, _cache, _rx) = create_execution_client_with_config(config);
20829 let credential = test_credential();
20830
20831 let window = i64::from(client.dispatch.nonce_manager.skip_window());
20832 for _ in 0..window {
20833 client
20834 .dispatch
20835 .nonce_manager
20836 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
20837 .unwrap();
20838 }
20839
20840 client
20843 .build_tx_context(&credential)
20844 .expect_err("window must trip");
20845 client
20846 .build_tx_context(&credential)
20847 .expect_err("window must still be exhausted");
20848
20849 wait_for_spawned_tasks(&client).await;
20850
20851 assert_eq!(
20852 hits.load(Ordering::Acquire),
20853 1,
20854 "burst exhaustion must trigger a single venue fetch",
20855 );
20856 assert!(!client.nonce_recovery_inflight.load(Ordering::Acquire));
20857 }
20858
20859 #[tokio::test]
20860 async fn refresh_nonce_releases_recovery_latch() {
20861 let venue_next_nonce = 77;
20862 let mut config = test_config();
20863 config.base_url_http = Some(spawn_next_nonce_server(venue_next_nonce).await);
20864 let (client, _cache, _rx) = create_execution_client_with_config(config);
20865
20866 client
20868 .nonce_recovery_inflight
20869 .store(true, Ordering::Release);
20870
20871 client.refresh_nonce().await.unwrap();
20872
20873 assert!(
20874 !client.nonce_recovery_inflight.load(Ordering::Acquire),
20875 "connect-time refresh must release a stuck recovery latch",
20876 );
20877 assert_eq!(
20878 client
20879 .dispatch
20880 .nonce_manager
20881 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
20882 .unwrap(),
20883 venue_next_nonce,
20884 "refresh must hard-reset allocation to the venue nonce",
20885 );
20886 }
20887
20888 #[tokio::test]
20889 async fn handle_send_tx_rejection_ack_create_emits_order_rejected() {
20890 let (client, cache, mut rx) = create_execution_client();
20891 let instrument_id = register_test_instrument(&client, &cache);
20892 let mut factory = test_order_factory();
20893 let order = test_limit_order(&mut factory, instrument_id, "REJECT-CREATE");
20894 let client_order_index = enqueue_create(&client, &order, 42);
20895
20896 handle_send_tx_rejection(
20897 &client.dispatch,
20898 &client.emitter,
20899 Some(TEST_ACCOUNT_INDEX_I64),
20900 UnixNanos::from(1_000_000_000),
20901 SendTxRejectionSource::Ack,
20902 Some(21702),
20903 "invalid price",
20904 None,
20905 );
20906
20907 let event = recv_order_event(&mut rx).await;
20908 match event {
20909 OrderEventAny::Rejected(e) => {
20910 assert_eq!(e.client_order_id, order.client_order_id());
20911 assert_eq!(e.reason, "LIGHTER_21702: invalid price");
20912 assert!(!e.due_post_only);
20913 }
20914 other => panic!("expected Rejected, was {other:?}"),
20915 }
20916
20917 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
20918 assert!(client.dispatch.cloid_map.get(&client_order_index).is_none());
20919 }
20920
20921 #[tokio::test]
20922 async fn handle_send_tx_rejection_ack_create_sets_due_post_only() {
20923 let (client, cache, mut rx) = create_execution_client();
20924 let instrument_id = register_test_instrument(&client, &cache);
20925 let mut factory = test_order_factory();
20926 let order = test_limit_order(&mut factory, instrument_id, "REJECT-POST-ONLY");
20927 enqueue_create(&client, &order, 43);
20928
20929 handle_send_tx_rejection(
20930 &client.dispatch,
20931 &client.emitter,
20932 Some(TEST_ACCOUNT_INDEX_I64),
20933 UnixNanos::from(1_000_000_000),
20934 SendTxRejectionSource::Ack,
20935 Some(21700),
20936 "post-only order would execute",
20937 None,
20938 );
20939
20940 let event = recv_order_event(&mut rx).await;
20941 match event {
20942 OrderEventAny::Rejected(e) => {
20943 assert_eq!(e.client_order_id, order.client_order_id());
20944 assert!(e.due_post_only);
20945 }
20946 other => panic!("expected Rejected, was {other:?}"),
20947 }
20948
20949 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
20950 }
20951
20952 #[rstest]
20953 #[case::explicit_enqueued(true)]
20954 #[case::explicit_waiting(false)]
20955 #[tokio::test]
20956 async fn unmarked_cancel_enqueue_preserves_explicit_rejection(#[case] explicit_enqueued: bool) {
20957 let (client, cache, mut rx) = create_execution_client();
20958 let instrument_id = register_test_instrument(&client, &cache);
20959 let mut factory = test_order_factory();
20960 let order = test_limit_order(&mut factory, instrument_id, "CANCEL-ELIGIBILITY");
20961 let id = order.client_order_id();
20962 let venue_id = VenueOrderId::from("123");
20963 cache_pending_cancel_order(&cache, order, venue_id);
20964 let context = client
20965 .fanout_dispatch_context(client.credential.as_ref().unwrap())
20966 .unwrap();
20967
20968 let plan = CancelOrderPlan {
20969 client_order_id: id,
20970 strategy_id: strategy_id(),
20971 instrument_id,
20972 venue_order_id: Some(venue_id),
20973 market_index: TEST_MARKET_INDEX,
20974 venue_index: 123,
20975 };
20976
20977 let first = context.sign_cancel_order(&plan).unwrap();
20978 let second = context.sign_cancel_order(&plan).unwrap();
20979 let (explicit, batch) = if explicit_enqueued {
20980 (first, second)
20981 } else {
20982 (second, first)
20983 };
20984
20985 client
20986 .dispatch
20987 .set_pending_order_action(id, PendingOrderAction::Cancel);
20988
20989 if explicit_enqueued {
20990 context.enqueue_cancel_order(&explicit, true);
20991 }
20992
20993 let mut batch = PreparedBatchTx::from((batch, false));
20994 batch.pending.batch_id = Some("batch-unmarked".to_string());
20995 context.enqueue_batch_tx(&batch);
20996 assert_eq!(
20997 client.dispatch.pending_cancel_nonce(&id),
20998 explicit_enqueued.then_some((0, explicit.nonce))
20999 );
21000
21001 if !explicit_enqueued {
21002 context.enqueue_cancel_order(&explicit, true);
21003 }
21004
21005 handle_send_tx_rejection_for_connection(
21006 &client.dispatch,
21007 &client.emitter,
21008 Some(TEST_ACCOUNT_INDEX_I64),
21009 0,
21010 UnixNanos::default(),
21011 SendTxRejectionSource::Ack,
21012 Some(21727),
21013 "unmarked batch failed",
21014 Some(&batch.pending.tx_hash),
21015 );
21016 assert_eq!(
21017 client.dispatch.pending_cancel_nonce(&id),
21018 Some((0, explicit.nonce))
21019 );
21020 assert!(rx.try_recv().is_err());
21021 handle_send_tx_rejection_for_connection(
21022 &client.dispatch,
21023 &client.emitter,
21024 Some(TEST_ACCOUNT_INDEX_I64),
21025 0,
21026 UnixNanos::default(),
21027 SendTxRejectionSource::Ack,
21028 Some(21727),
21029 "explicit cancel failed",
21030 Some(&explicit.tx_hash),
21031 );
21032 let event = recv_order_event(&mut rx).await;
21033 cache.borrow_mut().update_order(&event).unwrap();
21034 match event {
21035 OrderEventAny::CancelRejected(event) => {
21036 assert_eq!(event.client_order_id, id);
21037 assert_eq!(event.venue_order_id, Some(venue_id));
21038 assert_eq!(event.instrument_id, instrument_id);
21039 assert_eq!(event.strategy_id, strategy_id());
21040 assert_eq!(event.reason, "LIGHTER_21727: explicit cancel failed");
21041 }
21042 other => panic!("expected explicit cancel rejection, was {other:?}"),
21043 }
21044
21045 assert_eq!(
21046 cache.borrow().order(&id).unwrap().status(),
21047 OrderStatus::Accepted
21048 );
21049 assert_eq!(client.dispatch.pending_cancel_nonce(&id), None);
21050 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
21051 }
21052
21053 #[rstest]
21054 #[case::pending(PendingOrderAction::Cancel, true)]
21055 #[case::unmarked(PendingOrderAction::CancelUnmarked, false)]
21056 #[tokio::test]
21057 async fn cancel_rejection_preserves_newer_nonce(
21058 #[case] action: PendingOrderAction,
21059 #[case] emit_cancel_rejected: bool,
21060 ) {
21061 let (client, cache, mut rx) = create_execution_client();
21062 let instrument_id = register_test_instrument(&client, &cache);
21063 let id = ClientOrderId::from("CANCEL-NEWER");
21064
21065 let pending = PendingSendTx {
21066 batch_id: Some("batch-old".to_string()),
21067 connection_epoch: 3,
21068 kind: PendingSendTxKind::Cancel {
21069 emit_cancel_rejected,
21070 strategy_id: strategy_id(),
21071 instrument_id,
21072 client_order_id: id,
21073 venue_order_id: Some(VenueOrderId::from("123")),
21074 },
21075 submitted_at: UnixNanos::default(),
21076 nonce: 41,
21077 api_key_index: TEST_API_KEY_INDEX,
21078 tx_hash: "old-hash".to_string(),
21079 };
21080
21081 client.dispatch.set_pending_order_action(id, action);
21082 client
21083 .dispatch
21084 .set_pending_cancel_nonce(&id, 3, 42, emit_cancel_rejected);
21085 reject_cancel_order(
21086 &client.dispatch,
21087 &client.emitter,
21088 &pending,
21089 "old failure",
21090 UnixNanos::default(),
21091 );
21092 assert_eq!(client.dispatch.pending_cancel_nonce(&id), Some((3, 42)));
21093 assert_eq!(client.dispatch.pending_order_action(&id), Some(action));
21094 assert!(rx.try_recv().is_err());
21095 }
21096
21097 #[tokio::test]
21098 async fn handle_send_tx_rejection_cancel_emits_cancel_rejected() {
21099 let (client, cache, mut rx) = create_execution_client();
21100 let instrument_id = register_test_instrument(&client, &cache);
21101 let client_order_id = ClientOrderId::from("REJECT-CANCEL");
21102 let venue_order_id = VenueOrderId::from("123");
21103 let nonce = client
21104 .dispatch
21105 .nonce_manager
21106 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
21107 .unwrap();
21108
21109 client
21110 .dispatch
21111 .set_pending_order_action(client_order_id, PendingOrderAction::Cancel);
21112 client
21113 .dispatch
21114 .set_pending_cancel_nonce(&client_order_id, 0, nonce, true);
21115 client.dispatch.enqueue_pending_sendtx(PendingSendTx {
21116 batch_id: None,
21117 connection_epoch: 0,
21118 kind: PendingSendTxKind::Cancel {
21119 emit_cancel_rejected: true,
21120 strategy_id: strategy_id(),
21121 instrument_id,
21122 client_order_id,
21123 venue_order_id: Some(venue_order_id),
21124 },
21125 submitted_at: UnixNanos::from(1_000_000_000),
21126 nonce,
21127 api_key_index: TEST_API_KEY_INDEX,
21128 tx_hash: format!("hash{nonce:02x}"),
21129 });
21130
21131 handle_send_tx_rejection(
21132 &client.dispatch,
21133 &client.emitter,
21134 Some(TEST_ACCOUNT_INDEX_I64),
21135 UnixNanos::from(1_000_000_000),
21136 SendTxRejectionSource::Ack,
21137 Some(21727),
21138 "order is not cancelable",
21139 None,
21140 );
21141
21142 let event = recv_order_event(&mut rx).await;
21143 match event {
21144 OrderEventAny::CancelRejected(e) => {
21145 assert_eq!(e.client_order_id, client_order_id);
21146 assert_eq!(e.instrument_id, instrument_id);
21147 assert_eq!(e.venue_order_id, Some(venue_order_id));
21148 assert_eq!(e.reason, "LIGHTER_21727: order is not cancelable",);
21149 }
21150 other => panic!("expected CancelRejected, was {other:?}"),
21151 }
21152
21153 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
21154 assert_nonce_reusable(&client.dispatch);
21155 }
21156
21157 #[tokio::test]
21158 async fn handle_send_tx_rejection_modify_emits_modify_rejected() {
21159 let (client, cache, mut rx) = create_execution_client();
21160 let instrument_id = register_test_instrument(&client, &cache);
21161 let client_order_id = ClientOrderId::from("REJECT-MODIFY");
21162 let venue_order_id = VenueOrderId::from("456");
21163 let nonce = client
21164 .dispatch
21165 .nonce_manager
21166 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
21167 .unwrap();
21168
21169 enqueue_modify(
21170 &client,
21171 instrument_id,
21172 client_order_id,
21173 venue_order_id,
21174 nonce,
21175 );
21176
21177 handle_send_tx_rejection(
21178 &client.dispatch,
21179 &client.emitter,
21180 Some(TEST_ACCOUNT_INDEX_I64),
21181 UnixNanos::from(1_000_000_000),
21182 SendTxRejectionSource::Ack,
21183 Some(21702),
21184 "modify rejected by venue",
21185 None,
21186 );
21187
21188 let event = recv_order_event(&mut rx).await;
21189 match event {
21190 OrderEventAny::ModifyRejected(e) => {
21191 assert_eq!(e.client_order_id, client_order_id);
21192 assert_eq!(e.instrument_id, instrument_id);
21193 assert_eq!(e.venue_order_id, Some(venue_order_id));
21194 assert_eq!(e.reason, "LIGHTER_21702: modify rejected by venue",);
21195 }
21196 other => panic!("expected ModifyRejected, was {other:?}"),
21197 }
21198
21199 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
21200 assert_nonce_reusable(&client.dispatch);
21201 }
21202
21203 #[rstest]
21204 #[case::newer_modify(PendingOrderAction::Modify)]
21205 #[case::newer_cancel(PendingOrderAction::Cancel)]
21206 fn handle_send_tx_rejection_preserves_superseding_modify_action(
21207 #[case] action: PendingOrderAction,
21208 ) {
21209 let (client, cache, mut rx) = create_execution_client();
21210 let instrument = register_test_instrument(&client, &cache);
21211 let id = ClientOrderId::from("REJECT-MODIFY-SUPERSEDED");
21212 let venue = VenueOrderId::from("456");
21213 let nonce = client
21214 .dispatch
21215 .nonce_manager
21216 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
21217 .unwrap();
21218 enqueue_modify(&client, instrument, id, venue, nonce);
21219 let current_nonce = client
21220 .dispatch
21221 .nonce_manager
21222 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
21223 .unwrap();
21224
21225 assert!(client.dispatch.clear_pending_modify_if_nonce(&id, 0, nonce));
21226
21227 if action == PendingOrderAction::Modify {
21228 enqueue_modify(&client, instrument, id, venue, current_nonce);
21229 } else {
21230 client.dispatch.set_pending_order_action(id, action);
21231 client
21232 .dispatch
21233 .set_pending_cancel_nonce(&id, 0, current_nonce, true);
21234 }
21235
21236 handle_send_tx_rejection(
21237 &client.dispatch,
21238 &client.emitter,
21239 Some(TEST_ACCOUNT_INDEX_I64),
21240 UnixNanos::from(1_000_000_000),
21241 SendTxRejectionSource::Ack,
21242 Some(21702),
21243 "older modify rejected by venue",
21244 Some(&format!("hash{nonce:02x}")),
21245 );
21246
21247 assert!(rx.try_recv().is_err());
21248 assert_eq!(client.dispatch.pending_order_action(&id), Some(action));
21249 assert_eq!(
21250 client.dispatch.pending_modify_nonce(&id),
21251 (action == PendingOrderAction::Modify).then_some((0, current_nonce))
21252 );
21253 assert_eq!(
21254 client.dispatch.pending_cancel_nonce(&id),
21255 (action == PendingOrderAction::Cancel).then_some((0, current_nonce))
21256 );
21257 assert_eq!(
21258 client.dispatch.pending_sendtx_len(),
21259 usize::from(action == PendingOrderAction::Modify)
21260 );
21261 assert_eq!(
21262 client
21263 .dispatch
21264 .nonce_manager
21265 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
21266 .unwrap(),
21267 current_nonce + 1
21268 );
21269 }
21270
21271 #[tokio::test]
21272 async fn handle_send_tx_rejection_hash_match_attributes_past_the_head() {
21273 let (client, cache, mut rx) = create_execution_client();
21274 let instrument_id = register_test_instrument(&client, &cache);
21275 let mut factory = test_order_factory();
21276 let order_a = test_limit_order(&mut factory, instrument_id, "REJECT-HASH-A");
21277 let order_b = test_limit_order(&mut factory, instrument_id, "REJECT-HASH-B");
21278 enqueue_create(&client, &order_a, 10);
21279 enqueue_create(&client, &order_b, 11);
21280
21281 handle_send_tx_rejection(
21283 &client.dispatch,
21284 &client.emitter,
21285 Some(TEST_ACCOUNT_INDEX_I64),
21286 UnixNanos::from(1_000_000_000),
21287 SendTxRejectionSource::Ack,
21288 Some(21702),
21289 "invalid price",
21290 Some("hash0b"),
21291 );
21292
21293 let event = recv_order_event(&mut rx).await;
21294 match event {
21295 OrderEventAny::Rejected(e) => {
21296 assert_eq!(e.client_order_id, order_b.client_order_id());
21297 }
21298 other => panic!("expected Rejected, was {other:?}"),
21299 }
21300
21301 assert_eq!(client.dispatch.pending_sendtx_len(), 1, "A must survive");
21302 let head = client.dispatch.pop_pending_sendtx_head().unwrap();
21303 match head.kind {
21304 PendingSendTxKind::Create { order, .. } => {
21305 assert_eq!(order.client_order_id(), order_a.client_order_id());
21306 }
21307 _ => panic!("expected Create kind"),
21308 }
21309 }
21310
21311 #[tokio::test]
21312 async fn handle_send_tx_rejection_unmatched_hash_pops_nothing() {
21313 let (client, cache, mut rx) = create_execution_client();
21314 let instrument_id = register_test_instrument(&client, &cache);
21315 let mut factory = test_order_factory();
21316 let order = test_limit_order(&mut factory, instrument_id, "REJECT-UNMATCHED");
21317 enqueue_create(&client, &order, 10);
21318
21319 handle_send_tx_rejection(
21320 &client.dispatch,
21321 &client.emitter,
21322 Some(TEST_ACCOUNT_INDEX_I64),
21323 UnixNanos::from(1_000_000_000),
21324 SendTxRejectionSource::Ack,
21325 Some(21702),
21326 "invalid price",
21327 Some("0xbeef"),
21328 );
21329
21330 assert_eq!(
21331 client.dispatch.pending_sendtx_len(),
21332 1,
21333 "an echoed hash with no matching entry must not pop the head",
21334 );
21335 assert!(
21336 tokio::time::timeout(Duration::from_millis(50), rx.recv())
21337 .await
21338 .is_err(),
21339 "unmatched rejection must not emit an event",
21340 );
21341 }
21342
21343 #[tokio::test]
21344 async fn handle_send_tx_rejection_bare_error_within_window_attributes() {
21345 let (client, cache, mut rx) = create_execution_client();
21346 let instrument_id = register_test_instrument(&client, &cache);
21347 let mut factory = test_order_factory();
21348 let order = test_limit_order(&mut factory, instrument_id, "BARE-IN");
21349 enqueue_create(&client, &order, 50);
21350
21351 let within_window = UnixNanos::from(1_000_000_000 + 500 * 1_000_000);
21352 handle_send_tx_rejection(
21353 &client.dispatch,
21354 &client.emitter,
21355 Some(TEST_ACCOUNT_INDEX_I64),
21356 within_window,
21357 SendTxRejectionSource::BareError,
21358 Some(21149),
21359 "integrator is not approved",
21360 None,
21361 );
21362
21363 let event = recv_order_event(&mut rx).await;
21364 assert!(matches!(event, OrderEventAny::Rejected(_)));
21365 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
21366 }
21367
21368 #[tokio::test]
21369 async fn handle_send_tx_rejection_bare_error_outside_window_skips() {
21370 let (client, cache, mut rx) = create_execution_client();
21371 let instrument_id = register_test_instrument(&client, &cache);
21372 let mut factory = test_order_factory();
21373 let order = test_limit_order(&mut factory, instrument_id, "BARE-OUT");
21374 enqueue_create(&client, &order, 60);
21375
21376 let outside_window = UnixNanos::from(1_000_000_000 + 2_000 * 1_000_000);
21377 handle_send_tx_rejection(
21378 &client.dispatch,
21379 &client.emitter,
21380 Some(TEST_ACCOUNT_INDEX_I64),
21381 outside_window,
21382 SendTxRejectionSource::BareError,
21383 Some(99),
21384 "late error",
21385 None,
21386 );
21387
21388 assert_eq!(
21389 client.dispatch.pending_sendtx_len(),
21390 1,
21391 "head must remain queued past the 1s attribution window",
21392 );
21393 assert!(
21394 tokio::time::timeout(Duration::from_millis(50), rx.recv())
21395 .await
21396 .is_err(),
21397 "no event must be emitted outside the window",
21398 );
21399 }
21400
21401 #[tokio::test]
21402 async fn handle_send_tx_rejection_bare_error_with_multiple_pending_skips() {
21403 let (client, cache, mut rx) = create_execution_client();
21404 let instrument_id = register_test_instrument(&client, &cache);
21405 let mut factory = test_order_factory();
21406 let order_a = test_limit_order(&mut factory, instrument_id, "BARE-MULTI-A");
21407 let order_b = test_limit_order(&mut factory, instrument_id, "BARE-MULTI-B");
21408 enqueue_create(&client, &order_a, 70);
21409 enqueue_create(&client, &order_b, 71);
21410
21411 handle_send_tx_rejection(
21412 &client.dispatch,
21413 &client.emitter,
21414 Some(TEST_ACCOUNT_INDEX_I64),
21415 UnixNanos::from(1_000_000_000 + 500 * 1_000_000),
21416 SendTxRejectionSource::BareError,
21417 Some(21149),
21418 "integrator is not approved",
21419 None,
21420 );
21421
21422 assert_eq!(client.dispatch.pending_sendtx_len(), 2);
21423 assert!(
21424 tokio::time::timeout(Duration::from_millis(50), rx.recv())
21425 .await
21426 .is_err(),
21427 "a bare error cannot be attributed with multiple transactions pending",
21428 );
21429 }
21430
21431 #[tokio::test]
21432 async fn handle_send_tx_rejection_other_kind_rolls_back_latest_nonce() {
21433 let (client, _cache, _rx) = create_execution_client();
21434 let nonce = client
21435 .dispatch
21436 .nonce_manager
21437 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
21438 .unwrap();
21439 enqueue_other(&client, nonce);
21440
21441 let needs_resync = handle_send_tx_rejection(
21442 &client.dispatch,
21443 &client.emitter,
21444 Some(TEST_ACCOUNT_INDEX_I64),
21445 UnixNanos::from(1_000_000_000),
21446 SendTxRejectionSource::Ack,
21447 Some(23000),
21448 "Too Many Requests",
21449 None,
21450 );
21451
21452 assert!(!needs_resync, "rate-limit rejection must not force resync");
21453 assert_nonce_reusable(&client.dispatch);
21454 }
21455
21456 #[tokio::test]
21457 async fn handle_send_tx_rejection_other_kind_skips_rollback_with_newer_issuance() {
21458 let (client, _cache, _rx) = create_execution_client();
21459 let rejected_nonce = client
21460 .dispatch
21461 .nonce_manager
21462 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
21463 .unwrap();
21464 let newer_nonce = client
21465 .dispatch
21466 .nonce_manager
21467 .next_nonce(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX)
21468 .unwrap();
21469 enqueue_other(&client, rejected_nonce);
21470
21471 handle_send_tx_rejection(
21472 &client.dispatch,
21473 &client.emitter,
21474 Some(TEST_ACCOUNT_INDEX_I64),
21475 UnixNanos::from(1_000_000_000),
21476 SendTxRejectionSource::Ack,
21477 Some(23000),
21478 "Too Many Requests",
21479 None,
21480 );
21481
21482 assert_eq!(
21483 client
21484 .dispatch
21485 .nonce_manager
21486 .last_issued(TEST_ACCOUNT_INDEX_I64, TEST_API_KEY_INDEX),
21487 Some(newer_nonce),
21488 "non-latest rejection must leave last_issued alone",
21489 );
21490 }
21491
21492 #[tokio::test]
21493 async fn handle_send_tx_rejection_invalid_nonce_signals_resync() {
21494 let (client, _cache, _rx) = create_execution_client();
21495
21496 enqueue_other(&client, 70);
21497 let attributed = handle_send_tx_rejection(
21498 &client.dispatch,
21499 &client.emitter,
21500 Some(TEST_ACCOUNT_INDEX_I64),
21501 UnixNanos::from(1_000_000_000),
21502 SendTxRejectionSource::Ack,
21503 Some(LIGHTER_ERROR_CODE_INVALID_NONCE),
21504 "invalid nonce",
21505 None,
21506 );
21507 assert!(attributed, "attributed invalid nonce must signal resync");
21508
21509 let unattributed = handle_send_tx_rejection(
21510 &client.dispatch,
21511 &client.emitter,
21512 Some(TEST_ACCOUNT_INDEX_I64),
21513 UnixNanos::from(1_000_000_000),
21514 SendTxRejectionSource::Ack,
21515 Some(LIGHTER_ERROR_CODE_INVALID_NONCE),
21516 "invalid nonce",
21517 None,
21518 );
21519 assert!(
21520 unattributed,
21521 "unattributed invalid nonce must still signal resync",
21522 );
21523
21524 enqueue_other(&client, 71);
21525 let other_code = handle_send_tx_rejection(
21526 &client.dispatch,
21527 &client.emitter,
21528 Some(TEST_ACCOUNT_INDEX_I64),
21529 UnixNanos::from(1_000_000_000),
21530 SendTxRejectionSource::Ack,
21531 Some(21702),
21532 "invalid price",
21533 None,
21534 );
21535 assert!(!other_code, "other rejection codes must not force resync");
21536 }
21537
21538 #[tokio::test]
21539 async fn handle_send_tx_rejection_other_kind_logs_and_skips_emit() {
21540 let (client, _cache, mut rx) = create_execution_client();
21541 enqueue_other(&client, 70);
21542
21543 handle_send_tx_rejection(
21544 &client.dispatch,
21545 &client.emitter,
21546 Some(TEST_ACCOUNT_INDEX_I64),
21547 UnixNanos::from(1_000_000_000),
21548 SendTxRejectionSource::Ack,
21549 Some(21727),
21550 "invalid client order index",
21551 None,
21552 );
21553
21554 assert_eq!(client.dispatch.pending_sendtx_len(), 0, "Other head pops");
21555 assert!(
21556 tokio::time::timeout(Duration::from_millis(50), rx.recv())
21557 .await
21558 .is_err(),
21559 "Other-kind rejection must not emit OrderRejected",
21560 );
21561 }
21562
21563 #[tokio::test]
21564 async fn handle_send_tx_rejection_empty_queue_logs_warn() {
21565 let (client, _cache, mut rx) = create_execution_client();
21566
21567 handle_send_tx_rejection(
21568 &client.dispatch,
21569 &client.emitter,
21570 Some(TEST_ACCOUNT_INDEX_I64),
21571 UnixNanos::from(1_000_000_000),
21572 SendTxRejectionSource::Ack,
21573 Some(1),
21574 "no pending",
21575 None,
21576 );
21577
21578 assert_eq!(client.dispatch.pending_sendtx_len(), 0);
21579 assert!(
21580 tokio::time::timeout(Duration::from_millis(50), rx.recv())
21581 .await
21582 .is_err(),
21583 );
21584 }
21585
21586 #[rstest]
21587 #[case::standard_no_override(LighterAccountTier::Standard, None, 60, None)]
21588 #[case::standard_zero_is_default(LighterAccountTier::Standard, Some(0), 60, None)]
21589 #[case::standard_override_above_tier(
21590 LighterAccountTier::Standard,
21591 Some(24_000),
21592 24_000,
21593 Some(TierCrossCheck::AboveTier { documented: 60 })
21594 )]
21595 #[case::premium_raise_hint(
21596 LighterAccountTier::Premium,
21597 None,
21598 60,
21599 Some(TierCrossCheck::RaiseHint { documented: 24_000 })
21600 )]
21601 #[case::premium_configured_no_advisory(LighterAccountTier::Premium, Some(24_000), 24_000, None)]
21602 #[case::unknown_no_advisory(LighterAccountTier::Unknown(7), None, 60, None)]
21603 fn test_tier_quota_report(
21604 #[case] tier: LighterAccountTier,
21605 #[case] configured: Option<u32>,
21606 #[case] expected_active: u32,
21607 #[case] expected_cross_check: Option<TierCrossCheck>,
21608 ) {
21609 assert_eq!(
21610 tier_quota_report(tier, configured, 60),
21611 (expected_active, expected_cross_check),
21612 );
21613 }
21614
21615 #[rstest]
21616 fn cold_first_frame_updates_cached_shape() {
21617 let mut rig = dispatcher_rig("COLD-SHAPE");
21618 let mut frame = dispatcher_test_order(&rig, LighterOrderStatus::Open);
21619 rig.dispatch.forget_cloid(frame.client_order_index);
21620 let order = OrderTestBuilder::new(OrderType::Limit)
21621 .trader_id(trader_id())
21622 .strategy_id(strategy_id())
21623 .instrument_id(rig.instrument_id)
21624 .client_order_id(rig.cloid)
21625 .side(OrderSide::Buy)
21626 .quantity(Quantity::from("0.1000"))
21627 .price(Price::from("2352.74"))
21628 .build();
21629 let accepted =
21630 TestOrderEventStubs::accepted(&order, account_id(), VenueOrderId::new(&frame.order_id));
21631 let mut order = order;
21632 order.apply(accepted).unwrap();
21633 rig.dispatch.seed_order_restorations(vec![order]);
21634 frame.initial_base_amount = Decimal::from_str("0.0400").unwrap();
21635 frame.remaining_base_amount = frame.initial_base_amount;
21636 dispatch_lighter_order(
21637 &frame,
21638 &rig.dispatch,
21639 &rig.emitter,
21640 &rig.registry,
21641 account_id(),
21642 trader_id(),
21643 UnixNanos::from(2),
21644 );
21645 let events = drain_events(&mut rig.rx);
21646 assert_eq!(events.len(), 1);
21647
21648 let ExecutionEvent::Order(OrderEventAny::Updated(update)) = &events[0] else {
21649 panic!("expected restored shape update");
21650 };
21651
21652 assert_eq!(update.quantity, Quantity::from("0.0400"));
21653 assert_eq!(update.price, Some(Price::from("2352.74")));
21654 assert_eq!(update.client_order_id, rig.cloid);
21655 assert_eq!(
21656 update.venue_order_id,
21657 Some(VenueOrderId::new(&frame.order_id))
21658 );
21659 }
21660
21661 #[rstest]
21662 #[case(false)]
21663 #[case(true)]
21664 fn native_group_child_sizing_preserves_real_fill(#[case] trade_first: bool) {
21665 let mut rig = dispatcher_rig("CHILD-SIZING");
21666 let order = OrderTestBuilder::new(OrderType::StopLimit)
21667 .trader_id(trader_id())
21668 .strategy_id(strategy_id())
21669 .instrument_id(rig.instrument_id)
21670 .client_order_id(rig.cloid)
21671 .side(OrderSide::Buy)
21672 .quantity(Quantity::from("0.1000"))
21673 .price(Price::from("2352.74"))
21674 .trigger_price(Price::from("2300.00"))
21675 .parent_order_id(ClientOrderId::new("PARENT-CHILD-SIZING"))
21676 .reduce_only(true)
21677 .build();
21678 rig.dispatch.register_create_identity(&order).unwrap();
21679 rig.dispatch
21680 .mark_order_submission(&rig.cloid, TEST_SUBMISSION_NONCE);
21681 let mut frame = dispatcher_test_order(&rig, LighterOrderStatus::Pending);
21682 frame.initial_base_amount = Decimal::ZERO;
21683 frame.remaining_base_amount = Decimal::ZERO;
21684 frame.order_type = LighterOrderKind::StopLossLimit;
21685 frame.trigger_price = Decimal::from_str("2300.00").unwrap();
21686 frame.reduce_only = true;
21687 let cache = Rc::new(RefCell::new(Cache::default()));
21688 cache
21689 .borrow_mut()
21690 .add_instrument(
21691 LIGHTER_INSTRUMENT_CACHE
21692 .get(&rig.instrument_id)
21693 .unwrap()
21694 .value()
21695 .clone(),
21696 )
21697 .unwrap();
21698 cache
21699 .borrow_mut()
21700 .add_order(order, None, Some(client_id()), false)
21701 .unwrap();
21702 let clock: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(VirtualClock::default()));
21703 let mut engine = ExecutionEngine::new(clock, cache.clone(), None);
21704 dispatch_lighter_order(
21705 &frame,
21706 &rig.dispatch,
21707 &rig.emitter,
21708 &rig.registry,
21709 account_id(),
21710 trader_id(),
21711 UnixNanos::from(1),
21712 );
21713
21714 for event in drain_events(&mut rig.rx) {
21715 let ExecutionEvent::Order(event) = event else {
21716 panic!("expected acceptance");
21717 };
21718
21719 engine.process(&event);
21720 }
21721
21722 let mut trade = dispatcher_test_trade(&rig, true);
21723 trade.bid_id = frame.order_index;
21724 trade.bid_id_str = Some(frame.order_id.clone());
21725 trade.size = Decimal::from_str("0.0400").unwrap();
21726 trade.usd_amount = trade.size * trade.price;
21727 frame.status = LighterOrderStatus::Filled;
21728 frame.initial_base_amount = trade.size;
21729 frame.filled_base_amount = trade.size;
21730 frame.filled_quote_amount = trade.usd_amount;
21731 frame.order_type = LighterOrderKind::Limit;
21732 frame.trigger_price = Decimal::ZERO;
21733
21734 if trade_first {
21735 dispatch_lighter_trade(
21736 &trade,
21737 &rig.dispatch,
21738 &rig.emitter,
21739 &rig.registry,
21740 account_id(),
21741 trader_id(),
21742 Some(TEST_ACCOUNT_INDEX_I64),
21743 UnixNanos::from(2),
21744 );
21745 }
21746
21747 dispatch_lighter_order(
21748 &frame,
21749 &rig.dispatch,
21750 &rig.emitter,
21751 &rig.registry,
21752 account_id(),
21753 trader_id(),
21754 UnixNanos::from(3),
21755 );
21756
21757 if !trade_first {
21758 dispatch_lighter_trade(
21759 &trade,
21760 &rig.dispatch,
21761 &rig.emitter,
21762 &rig.registry,
21763 account_id(),
21764 trader_id(),
21765 Some(TEST_ACCOUNT_INDEX_I64),
21766 UnixNanos::from(4),
21767 );
21768 }
21769
21770 let events = drain_events(&mut rig.rx);
21771 for event in events {
21772 match event {
21773 ExecutionEvent::Order(event) => engine.process(&event),
21774 ExecutionEvent::Report(report) => engine.reconcile_execution_report(&report),
21775 _ => panic!("unexpected child event"),
21776 }
21777 }
21778
21779 let cache = cache.borrow();
21780 let order = cache.order(&rig.cloid).unwrap();
21781
21782 let fills = order
21783 .events()
21784 .into_iter()
21785 .filter_map(|event| match event {
21786 OrderEventAny::Filled(fill) => Some(fill),
21787 _ => None,
21788 })
21789 .collect::<Vec<_>>();
21790
21791 assert_eq!(order.status(), OrderStatus::Filled);
21792 assert_eq!(order.quantity(), Quantity::from("0.0400"));
21793 assert_eq!(order.filled_qty(), Quantity::from("0.0400"));
21794 assert_eq!(order.leaves_qty(), Quantity::from("0.0000"));
21795 assert_eq!(order.is_triggered(), Some(true));
21796 assert_eq!(fills.len(), 1);
21797 assert_eq!(fills[0].trade_id, parse_lighter_trade_id(&trade).unwrap());
21798 assert_eq!(fills[0].last_qty, Quantity::from("0.0400"));
21799 assert_eq!(fills[0].last_px, Price::from("2352.73"));
21800 assert_eq!(fills[0].commission, Some(Money::from("0.00263506 USDC")));
21801 }
21802
21803 #[rstest]
21804 #[case::distinct_timestamps(false)]
21805 #[case::same_timestamp(true)]
21806 fn native_group_preserves_position_fill_order(#[case] same_timestamp: bool) {
21807 let mut rig = dispatcher_rig("POSITION-PARENT");
21808 let parent_id = rig.cloid;
21809 let child_id = ClientOrderId::new("POSITION-CHILD");
21810 let extra_id = ClientOrderId::new("POSITION-EXTRA");
21811 let parent = OrderTestBuilder::new(OrderType::Limit)
21812 .trader_id(trader_id())
21813 .strategy_id(strategy_id())
21814 .instrument_id(rig.instrument_id)
21815 .client_order_id(parent_id)
21816 .side(OrderSide::Buy)
21817 .quantity(Quantity::from("0.1000"))
21818 .price(Price::from("100.00"))
21819 .build();
21820 rig.dispatch.register_create_identity(&parent).unwrap();
21821 rig.dispatch
21822 .mark_order_submission(&parent_id, TEST_SUBMISSION_NONCE);
21823 let mut parent_frame = dispatcher_test_order(&rig, LighterOrderStatus::Open);
21824 parent_frame.order_index = 562_947_905_631_053;
21825 parent_frame.order_id = parent_frame.order_index.to_string();
21826 parent_frame.initial_base_amount = Decimal::from_str("0.1000").unwrap();
21827 parent_frame.remaining_base_amount = parent_frame.initial_base_amount;
21828 parent_frame.price = Decimal::from_str("100.00").unwrap();
21829 dispatch_lighter_order(
21830 &parent_frame,
21831 &rig.dispatch,
21832 &rig.emitter,
21833 &rig.registry,
21834 account_id(),
21835 trader_id(),
21836 UnixNanos::from(1),
21837 );
21838 let mut opening = dispatcher_test_trade(&rig, true);
21839 opening.bid_id = parent_frame.order_index;
21840 opening.bid_id_str = Some(parent_frame.order_id);
21841 opening.trade_id_str = Some("900001".to_string());
21842 opening.size = Decimal::from_str("0.1000").unwrap();
21843 opening.price = Decimal::from_str("100.00").unwrap();
21844 opening.usd_amount = opening.size * opening.price;
21845 opening.maker_fee = Some(0);
21846 opening.taker_fee = Some(0);
21847 dispatch_lighter_trade(
21848 &opening,
21849 &rig.dispatch,
21850 &rig.emitter,
21851 &rig.registry,
21852 account_id(),
21853 trader_id(),
21854 Some(TEST_ACCOUNT_INDEX_I64),
21855 UnixNanos::from(2),
21856 );
21857 rig.cloid = child_id;
21858 let child = OrderTestBuilder::new(OrderType::StopLimit)
21859 .trader_id(trader_id())
21860 .strategy_id(strategy_id())
21861 .instrument_id(rig.instrument_id)
21862 .client_order_id(child_id)
21863 .side(OrderSide::Sell)
21864 .quantity(Quantity::from("0.1000"))
21865 .price(Price::from("200.00"))
21866 .trigger_price(Price::from("150.00"))
21867 .parent_order_id(parent_id)
21868 .reduce_only(true)
21869 .build();
21870 rig.dispatch.register_create_identity(&child).unwrap();
21871 rig.dispatch
21872 .mark_order_submission(&child_id, TEST_SUBMISSION_NONCE);
21873 let mut child_frame = dispatcher_test_order(&rig, LighterOrderStatus::Open);
21874 child_frame.initial_base_amount = Decimal::from_str("0.0400").unwrap();
21875 child_frame.remaining_base_amount = child_frame.initial_base_amount;
21876 child_frame.price = Decimal::from_str("200.00").unwrap();
21877 child_frame.side = Some(LighterOrderSide::Sell);
21878 child_frame.is_ask = true;
21879 child_frame.reduce_only = true;
21880 dispatch_lighter_order(
21881 &child_frame,
21882 &rig.dispatch,
21883 &rig.emitter,
21884 &rig.registry,
21885 account_id(),
21886 trader_id(),
21887 UnixNanos::from(3),
21888 );
21889 let mut first = dispatcher_test_trade(&rig, false);
21890 first.ask_id = child_frame.order_index;
21891 first.ask_id_str = Some(child_frame.order_id.clone());
21892 first.trade_id_str = Some("900002".to_string());
21893 first.timestamp = opening.timestamp + i64::from(!same_timestamp);
21894 first.size = Decimal::from_str("0.0200").unwrap();
21895 first.price = Decimal::from_str("200.00").unwrap();
21896 first.usd_amount = first.size * first.price;
21897 first.maker_fee = Some(0);
21898 first.taker_fee = Some(0);
21899 dispatch_lighter_trade(
21900 &first,
21901 &rig.dispatch,
21902 &rig.emitter,
21903 &rig.registry,
21904 account_id(),
21905 trader_id(),
21906 Some(TEST_ACCOUNT_INDEX_I64),
21907 UnixNanos::from(4),
21908 );
21909 rig.cloid = extra_id;
21910 let extra = OrderTestBuilder::new(OrderType::Limit)
21911 .trader_id(trader_id())
21912 .strategy_id(strategy_id())
21913 .instrument_id(rig.instrument_id)
21914 .client_order_id(extra_id)
21915 .side(OrderSide::Buy)
21916 .quantity(Quantity::from("0.1000"))
21917 .price(Price::from("100.00"))
21918 .time_in_force(TimeInForce::Ioc)
21919 .build();
21920 rig.dispatch.register_create_identity(&extra).unwrap();
21921 rig.dispatch
21922 .mark_order_submission(&extra_id, TEST_SUBMISSION_NONCE + 1);
21923 let mut extra_frame = dispatcher_test_order(&rig, LighterOrderStatus::Open);
21924 extra_frame.order_index += 5;
21925 extra_frame.order_id = extra_frame.order_index.to_string();
21926 extra_frame.initial_base_amount = Decimal::from_str("0.1000").unwrap();
21927 extra_frame.remaining_base_amount = extra_frame.initial_base_amount;
21928 extra_frame.price = Decimal::from_str("100.00").unwrap();
21929 dispatch_lighter_order(
21930 &extra_frame,
21931 &rig.dispatch,
21932 &rig.emitter,
21933 &rig.registry,
21934 account_id(),
21935 trader_id(),
21936 UnixNanos::from(5),
21937 );
21938 let mut addition = dispatcher_test_trade(&rig, true);
21939 addition.bid_id = extra_frame.order_index;
21940 addition.bid_id_str = Some(extra_frame.order_id);
21941 addition.trade_id_str = Some("900003".to_string());
21942 addition.timestamp = opening.timestamp + 2 * i64::from(!same_timestamp);
21943 addition.size = Decimal::from_str("0.0600").unwrap();
21944 addition.price = Decimal::from_str("50.00").unwrap();
21945 addition.usd_amount = addition.size * addition.price;
21946 addition.maker_fee = Some(0);
21947 addition.taker_fee = Some(0);
21948 dispatch_lighter_trade(
21949 &addition,
21950 &rig.dispatch,
21951 &rig.emitter,
21952 &rig.registry,
21953 account_id(),
21954 trader_id(),
21955 Some(TEST_ACCOUNT_INDEX_I64),
21956 UnixNanos::from(6),
21957 );
21958 rig.cloid = child_id;
21959 let mut last = first;
21960 last.trade_id_str = Some("900004".to_string());
21961 last.timestamp = opening.timestamp + 3 * i64::from(!same_timestamp);
21962 child_frame.status = LighterOrderStatus::Filled;
21963 child_frame.remaining_base_amount = Decimal::ZERO;
21964 child_frame.filled_base_amount = Decimal::from_str("0.0400").unwrap();
21965 child_frame.filled_quote_amount = Decimal::from(8);
21966 dispatch_lighter_order(
21967 &child_frame,
21968 &rig.dispatch,
21969 &rig.emitter,
21970 &rig.registry,
21971 account_id(),
21972 trader_id(),
21973 UnixNanos::from(7),
21974 );
21975 dispatch_lighter_trade(
21976 &last,
21977 &rig.dispatch,
21978 &rig.emitter,
21979 &rig.registry,
21980 account_id(),
21981 trader_id(),
21982 Some(TEST_ACCOUNT_INDEX_I64),
21983 UnixNanos::from(8),
21984 );
21985 let mut fills = Vec::new();
21986
21987 for event in drain_events(&mut rig.rx) {
21988 match event {
21989 ExecutionEvent::Order(OrderEventAny::Filled(fill)) => fills.push(fill),
21990 ExecutionEvent::Report(EngineExecutionReport::OrderWithFills(_, reports)) => {
21991 for report in reports {
21992 fills.push(OrderFilled::new(
21993 trader_id(),
21994 strategy_id(),
21995 rig.instrument_id,
21996 report.client_order_id.unwrap(),
21997 report.venue_order_id,
21998 account_id(),
21999 report.trade_id,
22000 report.order_side,
22001 OrderType::StopLimit,
22002 report.last_qty,
22003 report.last_px,
22004 Currency::from("USDC"),
22005 report.liquidity_side,
22006 UUID4::new(),
22007 report.ts_event,
22008 report.ts_init,
22009 true,
22010 None,
22011 Some(report.commission),
22012 None,
22013 ));
22014 }
22015 }
22016 _ => {}
22017 }
22018 }
22019
22020 let ids = fills.iter().map(|fill| fill.trade_id).collect::<Vec<_>>();
22021 let position_id = PositionId::new("POSITION-ORDER");
22022 for fill in &mut fills {
22023 fill.position_id = Some(position_id);
22024 }
22025
22026 let instrument = LIGHTER_INSTRUMENT_CACHE.get(&rig.instrument_id).unwrap();
22027 let mut position = Position::new(&instrument, fills[0].clone());
22028 for fill in &fills[1..] {
22029 position.apply(fill);
22030 }
22031
22032 let expected_pnl = Money::from_decimal_native(
22034 Decimal::from(31) / Decimal::from(7),
22035 Currency::from("USDC"),
22036 )
22037 .unwrap();
22038
22039 assert_eq!(
22040 ids,
22041 ["900001", "900002", "900003", "900004"].map(TradeId::new)
22042 );
22043 assert_eq!(position.quantity, Quantity::from("0.1200"));
22044 assert_eq!(position.realized_pnl, Some(expected_pnl));
22045 }
22046
22047 #[rstest]
22048 #[case::filled(LighterOrderStatus::Filled)]
22049 #[case::partial_then_canceled(LighterOrderStatus::Canceled)]
22050 fn terminal_after_fill_retires_identity_with_publication(#[case] status: LighterOrderStatus) {
22051 let mut rig = dispatcher_rig("TERMINAL-PUBLICATION");
22052 register_identity(&rig);
22053 let mut frame = dispatcher_test_order(&rig, LighterOrderStatus::Open);
22054 dispatch_lighter_order(
22055 &frame,
22056 &rig.dispatch,
22057 &rig.emitter,
22058 &rig.registry,
22059 account_id(),
22060 trader_id(),
22061 UnixNanos::from(1),
22062 );
22063 let mut trade = dispatcher_test_trade(&rig, true);
22064 trade.bid_id = frame.order_index;
22065 trade.bid_id_str = Some(frame.order_id.clone());
22066 trade.size = if status == LighterOrderStatus::Filled {
22067 frame.initial_base_amount
22068 } else {
22069 frame.initial_base_amount / Decimal::from(2)
22070 };
22071
22072 trade.usd_amount = trade.size * trade.price;
22073 dispatch_lighter_trade(
22074 &trade,
22075 &rig.dispatch,
22076 &rig.emitter,
22077 &rig.registry,
22078 account_id(),
22079 trader_id(),
22080 Some(TEST_ACCOUNT_INDEX_I64),
22081 UnixNanos::from(2),
22082 );
22083 frame.status = status;
22084 frame.filled_base_amount = trade.size;
22085 frame.remaining_base_amount = Decimal::ZERO;
22086 dispatch_lighter_order(
22087 &frame,
22088 &rig.dispatch,
22089 &rig.emitter,
22090 &rig.registry,
22091 account_id(),
22092 trader_id(),
22093 UnixNanos::from(3),
22094 );
22095 let identity = rig.dispatch.order_identity(&rig.cloid).unwrap();
22096 let events = drain_events(&mut rig.rx);
22097 assert!(!rig.dispatch.order_identities.contains_key(&rig.cloid));
22098 assert!(!rig.dispatch.venue_id_map.contains_key(&rig.cloid));
22099 assert!(
22100 !rig.dispatch
22101 .cloid_map
22102 .contains_key(&frame.client_order_index)
22103 );
22104 assert!(!rig.dispatch.order_snapshots.contains_key(&rig.cloid));
22105 assert!(identity.has_publication());
22106 assert!(
22107 rig.dispatch
22108 .any_order_identity(|id, other, _| *id == rig.cloid && other.has_publication())
22109 );
22110 assert_eq!(
22111 events
22112 .iter()
22113 .filter(|e| matches!(e, ExecutionEvent::Order(OrderEventAny::Filled(_))))
22114 .count(),
22115 1
22116 );
22117 }
22118
22119 #[rstest]
22120 #[case::modify("modify", false)]
22121 #[case::terminal("terminal", false)]
22122 #[case::fill("fill", false)]
22123 #[case::update("update", false)]
22124 #[case::retired_fill("fill", true)]
22125 #[case::retired_update("update", true)]
22126 fn position_reports_wait_for_execution(#[case] hold: &str, #[case] retired: bool) {
22127 let (client, cache, _) = create_execution_client();
22128 let instrument_id = register_test_instrument(&client, &cache);
22129 let mut factory = test_order_factory();
22130 let order = test_limit_order(&mut factory, instrument_id, "POSITION-HOLD");
22131 let cloid = order.client_order_id();
22132 client.dispatch.register_create_identity(&order).unwrap();
22133 let identity = client.dispatch.order_identity(&cloid).unwrap();
22134
22135 match hold {
22136 "modify" => {
22137 enqueue_modify(&client, instrument_id, cloid, VenueOrderId::new("700"), 23);
22138 }
22139 "terminal" => {
22140 let rig = dispatcher_rig("POSITION-TERMINAL");
22141 client
22142 .dispatch
22143 .order_events
22144 .lock()
22145 .entry(cloid)
22146 .or_default()
22147 .order = Some(dispatcher_test_order(&rig, LighterOrderStatus::Canceled));
22148 }
22149 "update" => identity.publish_update(UUID4::new()),
22150 "fill" => {
22151 let instrument = cache.borrow().instrument(&instrument_id).unwrap().clone();
22152
22153 let OrderEventAny::Filled(fill) = TestOrderEventStubs::filled(
22154 &order,
22155 &instrument,
22156 Some(TradeId::new("HELD-FILL")),
22157 None,
22158 Some(Price::from("2360.00")),
22159 Some(Quantity::from("0.0050")),
22160 None,
22161 Some(Money::from("0.01000000 USDC")),
22162 None,
22163 Some(account_id()),
22164 ) else {
22165 unreachable!()
22166 };
22167
22168 identity.publish_fill(&fill);
22169 }
22170 _ => unreachable!(),
22171 }
22172
22173 if retired {
22174 client.dispatch.retire_order_identity(&cloid);
22175 }
22176
22177 let report = PositionStatusReport::new(
22178 account_id(),
22179 instrument_id,
22180 PositionSide::Long,
22181 Quantity::from("0.0050"),
22182 UnixNanos::from(4),
22183 UnixNanos::from(5),
22184 None,
22185 None,
22186 Some(Decimal::from_str("2360.00").unwrap()),
22187 );
22188 client
22189 .dispatch
22190 .replace_position_snapshot(std::slice::from_ref(&report), &[], &[]);
22191
22192 let cmd = GeneratePositionStatusReports::new(
22193 UUID4::new(),
22194 UnixNanos::from(6),
22195 Some(instrument_id),
22196 None,
22197 None,
22198 None,
22199 None,
22200 );
22201 let (reports, complete, coverage) = client.cached_position_reports(&cmd).unwrap();
22202 assert!(reports.is_empty());
22203 assert!(!complete);
22204 assert_eq!(coverage, Some(AHashSet::from_iter([TEST_MARKET_INDEX])));
22205 let other_id = client.registry.insert(1, "BTC", LighterProductType::Perp);
22206 let mut other_cmd = cmd;
22207 other_cmd.instrument_id = Some(other_id);
22208 let (reports, complete, _) = client.cached_position_reports(&other_cmd).unwrap();
22209 assert!(reports.is_empty());
22210 assert!(complete);
22211 }
22212
22213 #[rstest]
22214 #[case::explicit_flat(true)]
22215 #[case::empty_snapshot(false)]
22216 #[tokio::test]
22217 async fn restart_held_child_fill_does_not_synthesize_position(#[case] explicit_flat: bool) {
22218 let (mut client, cache, mut rx) = create_execution_client();
22219 let core = client.core.clone();
22220 let instrument_id = register_test_instrument(&client, &cache);
22221 let mut rig = dispatcher_rig("RESTART-CHILD");
22222 rig.dispatch = client.dispatch.clone();
22223 rig.registry = Arc::clone(&client.registry);
22224
22225 let account = MarginAccount::new(
22226 AccountState::new(
22227 account_id(),
22228 AccountType::Margin,
22229 vec![AccountBalance::new(
22230 Money::from("10000.00000000 USDC"),
22231 Money::from("0.00000000 USDC"),
22232 Money::from("10000.00000000 USDC"),
22233 )],
22234 Vec::new(),
22235 true,
22236 UUID4::new(),
22237 UnixNanos::from(1),
22238 UnixNanos::from(1),
22239 Some(Currency::from("USDC")),
22240 ),
22241 true,
22242 );
22243 cache
22244 .borrow_mut()
22245 .add_account(AccountAny::Margin(account))
22246 .unwrap();
22247 let parent_id = ClientOrderId::new("RESTART-PARENT");
22248 let position_id = PositionId::new("RESTART-POSITION");
22249 let parent = OrderTestBuilder::new(OrderType::Limit)
22250 .trader_id(trader_id())
22251 .strategy_id(strategy_id())
22252 .instrument_id(instrument_id)
22253 .client_order_id(parent_id)
22254 .side(OrderSide::Buy)
22255 .quantity(Quantity::from("0.1000"))
22256 .price(Price::from("2352.73"))
22257 .build();
22258 let instrument = LIGHTER_INSTRUMENT_CACHE
22259 .get(&instrument_id)
22260 .unwrap()
22261 .value()
22262 .clone();
22263 let opening = TestOrderEventStubs::filled(
22264 &parent,
22265 &instrument,
22266 Some(TradeId::new("RESTART-OPENING")),
22267 Some(position_id),
22268 Some(Price::from("2352.73")),
22269 Some(Quantity::from("0.1000")),
22270 Some(LiquiditySide::Taker),
22271 Some(Money::from("0.00000000 USDC")),
22272 Some(UnixNanos::from(1)),
22273 Some(account_id()),
22274 );
22275 let mut parent = parent;
22276 parent
22277 .apply(TestOrderEventStubs::accepted(
22278 &parent,
22279 account_id(),
22280 VenueOrderId::new("600"),
22281 ))
22282 .unwrap();
22283 parent.apply(opening.clone()).unwrap();
22284 cache
22285 .borrow_mut()
22286 .add_order(parent, Some(position_id), Some(client_id()), false)
22287 .unwrap();
22288
22289 let OrderEventAny::Filled(opening) = opening else {
22290 unreachable!()
22291 };
22292
22293 let position = Position::new(&instrument, opening);
22294 cache
22295 .borrow_mut()
22296 .add_position(&position, OmsType::Netting)
22297 .unwrap();
22298 let child = OrderTestBuilder::new(OrderType::StopLimit)
22299 .trader_id(trader_id())
22300 .strategy_id(strategy_id())
22301 .instrument_id(instrument_id)
22302 .client_order_id(rig.cloid)
22303 .side(OrderSide::Sell)
22304 .quantity(Quantity::from("0.1000"))
22305 .price(Price::from("2352.74"))
22306 .trigger_price(Price::from("2300.00"))
22307 .parent_order_id(parent_id)
22308 .reduce_only(true)
22309 .build();
22310 let mut frame = dispatcher_test_order(&rig, LighterOrderStatus::Open);
22311 frame.side = Some(LighterOrderSide::Sell);
22312 frame.is_ask = true;
22313 frame.order_type = LighterOrderKind::Limit;
22314 frame.reduce_only = true;
22315 let venue_id = VenueOrderId::new(&frame.order_id);
22316 cache_accepted_order(&cache, child, venue_id, Some(client_id()));
22317 cache
22318 .borrow_mut()
22319 .add_position_id(
22320 &position_id,
22321 &instrument_id.venue,
22322 &rig.cloid,
22323 &strategy_id(),
22324 )
22325 .unwrap();
22326 let child = cache.borrow().order_owned(&rig.cloid).unwrap();
22327 client
22328 .dispatch
22329 .restore_reconciled_order(&child, frame.client_order_index, venue_id, false)
22330 .unwrap();
22331 let mut trade = dispatcher_test_trade(&rig, false);
22332 trade.ask_id = frame.order_index;
22333 trade.ask_id_str = Some(frame.order_id.clone());
22334 trade.size = Decimal::from_str("0.1000").unwrap();
22335 trade.usd_amount = trade.size * trade.price;
22336 trade.maker_fee = Some(0);
22337 trade.taker_fee = Some(0);
22338 dispatch_lighter_trade(
22339 &trade,
22340 &client.dispatch,
22341 &client.emitter,
22342 &client.registry,
22343 account_id(),
22344 trader_id(),
22345 Some(TEST_ACCOUNT_INDEX_I64),
22346 UnixNanos::from(2),
22347 );
22348
22349 let flat = PositionStatusReport::new(
22350 account_id(),
22351 instrument_id,
22352 PositionSide::Flat,
22353 Quantity::from("0.0000"),
22354 UnixNanos::from(3),
22355 UnixNanos::from(4),
22356 None,
22357 None,
22358 None,
22359 );
22360
22361 let positions = if explicit_flat {
22362 vec![flat]
22363 } else {
22364 Vec::new()
22365 };
22366
22367 client
22368 .dispatch
22369 .replace_position_snapshot(&positions, &[], &[]);
22370 let mut external = frame.clone();
22371 external.order_index += 1;
22372 external.order_id = external.order_index.to_string();
22373 external.client_order_index = 0;
22374 external.client_order_id = "0".to_string();
22375 external.reduce_only = false;
22376 let active = serde_json::json!({"code":200,"orders":[external]});
22377 let app = Router::new()
22378 .route(
22379 "/api/v1/accountActiveOrders",
22380 get(move || {
22381 let active = active.clone();
22382 async move { axum::Json(active) }
22383 }),
22384 )
22385 .route(
22386 "/api/v1/accountInactiveOrders",
22387 get(|| async { axum::Json(serde_json::json!({"code":200,"orders":[]})) }),
22388 )
22389 .route(
22390 "/api/v1/trades",
22391 get(|| async { axum::Json(serde_json::json!({"code":200,"trades":[]})) }),
22392 );
22393
22394 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
22395 let addr = listener.local_addr().unwrap();
22396 let server = tokio::spawn(async move {
22397 axum::serve(listener, app).await.unwrap();
22398 });
22399
22400 client.http_client = LighterHttpClient::new(
22401 LighterEnvironment::Testnet,
22402 Some(format!("http://{addr}")),
22403 1,
22404 None,
22405 )
22406 .unwrap();
22407 let mass = client
22408 .generate_mass_status(Some(60))
22409 .await
22410 .unwrap()
22411 .unwrap();
22412 server.abort();
22413 let _ = server.await;
22414
22415 let command = GeneratePositionStatusReports::new(
22416 UUID4::new(),
22417 UnixNanos::from(5),
22418 None,
22419 None,
22420 None,
22421 None,
22422 None,
22423 );
22424 let direct = client.generate_position_status_reports(&command).await;
22425 let worker = client
22426 .generate_position_status_reports_task(&command)
22427 .unwrap();
22428 worker.collection.await;
22429 let worker = (worker.finish)();
22430 assert!(mass.position_reports().is_empty());
22431 assert!(!mass.reports_complete());
22432 assert!(direct.is_err());
22433 assert!(worker.is_err());
22434 assert!(drain_events(&mut rx).is_empty());
22435 let clock: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(VirtualClock::default()));
22436
22437 let engine = Rc::new(RefCell::new(ExecutionEngine::new(
22438 clock.clone(),
22439 cache.clone(),
22440 None,
22441 )));
22442 let mut manager =
22443 ExecutionManager::new(clock, cache.clone(), ExecutionManagerConfig::default()).unwrap();
22444 let result = manager.reconcile_execution_mass_status(&mass, &engine);
22445 assert!(
22446 result
22447 .events
22448 .iter()
22449 .all(|event| !matches!(event, OrderEventAny::Filled(_)))
22450 );
22451 assert_eq!(
22452 cache.borrow().position(&position_id).unwrap().quantity,
22453 Quantity::from("0.1000")
22454 );
22455 frame.status = LighterOrderStatus::Filled;
22456 frame.initial_base_amount = trade.size;
22457 frame.filled_base_amount = trade.size;
22458 frame.remaining_base_amount = Decimal::ZERO;
22459 frame.filled_quote_amount = trade.usd_amount;
22460 dispatch_lighter_order(
22461 &frame,
22462 &client.dispatch,
22463 &client.emitter,
22464 &client.registry,
22465 account_id(),
22466 trader_id(),
22467 UnixNanos::from(6),
22468 );
22469 engine
22470 .borrow_mut()
22471 .register_client(Box::new(client))
22472 .unwrap();
22473 let released = drain_events(&mut rx);
22474 assert!(
22475 released
22476 .iter()
22477 .any(|event| matches!(event, ExecutionEvent::Order(OrderEventAny::Filled(_)))),
22478 "released child events: {released:?}"
22479 );
22480
22481 for event in released {
22482 match event {
22483 ExecutionEvent::Order(event) => engine.borrow_mut().process(&event),
22484 ExecutionEvent::Report(report) => {
22485 engine.borrow_mut().reconcile_execution_report(&report);
22486 }
22487 _ => panic!("unexpected restart event"),
22488 }
22489 }
22490
22491 let cache_ref = cache.borrow();
22492 let position = cache_ref.position(&position_id).unwrap();
22493 let child = cache_ref.order(&rig.cloid).unwrap();
22494 assert_eq!(position.quantity, Quantity::from("0.0000"));
22495 assert_eq!(position.realized_pnl, Some(Money::from("0.00000000 USDC")));
22496 assert_eq!(
22497 child.trade_ids(),
22498 vec![&parse_lighter_trade_id(&trade).unwrap()]
22499 );
22500 assert_eq!(child.filled_qty(), Quantity::from("0.1000"));
22501 assert!(!rig.dispatch.order_identities.contains_key(&rig.cloid));
22502 assert!(!rig.dispatch.venue_id_map.contains_key(&rig.cloid));
22503
22504 let command = GeneratePositionStatusReports::new(
22505 UUID4::new(),
22506 UnixNanos::from(7),
22507 None,
22508 None,
22509 None,
22510 None,
22511 None,
22512 );
22513 let (reports, complete, coverage) =
22514 cached_position_reports(&core, &rig.dispatch, &rig.registry, &command).unwrap();
22515 assert!(complete);
22516 assert_eq!(coverage, Some(AHashSet::new()));
22517 assert_eq!(reports.len(), usize::from(explicit_flat));
22518 }
22519
22520 #[rstest]
22521 fn resting_child_does_not_withhold_position_reports() {
22522 let (client, cache, _) = create_execution_client();
22523 let instrument_id = register_test_instrument(&client, &cache);
22524 let rig = dispatcher_rig("RESTING-POSITION");
22525 let order = OrderTestBuilder::new(OrderType::StopLimit)
22526 .trader_id(trader_id())
22527 .strategy_id(strategy_id())
22528 .instrument_id(instrument_id)
22529 .client_order_id(rig.cloid)
22530 .side(OrderSide::Sell)
22531 .quantity(Quantity::from("0.1000"))
22532 .price(Price::from("2300.00"))
22533 .trigger_price(Price::from("2301.00"))
22534 .parent_order_id(ClientOrderId::new("RESTING-PARENT"))
22535 .reduce_only(true)
22536 .build();
22537 client.dispatch.register_create_identity(&order).unwrap();
22538 client
22539 .dispatch
22540 .order_events
22541 .lock()
22542 .entry(rig.cloid)
22543 .or_default()
22544 .order = Some(dispatcher_test_order(&rig, LighterOrderStatus::Pending));
22545
22546 let report = PositionStatusReport::new(
22547 account_id(),
22548 instrument_id,
22549 PositionSide::Long,
22550 Quantity::from("0.1000"),
22551 UnixNanos::from(4),
22552 UnixNanos::from(5),
22553 None,
22554 None,
22555 Some(Decimal::from_str("2360.00").unwrap()),
22556 );
22557 client
22558 .dispatch
22559 .replace_position_snapshot(std::slice::from_ref(&report), &[], &[]);
22560
22561 let cmd = GeneratePositionStatusReports::new(
22562 UUID4::new(),
22563 UnixNanos::from(6),
22564 None,
22565 None,
22566 None,
22567 None,
22568 None,
22569 );
22570 let (reports, complete, coverage) = client.cached_position_reports(&cmd).unwrap();
22571 assert_eq!(reports, vec![report]);
22572 assert!(complete);
22573 assert_eq!(coverage, Some(AHashSet::new()));
22574 }
22575
22576 #[rstest]
22577 #[case::cached_fill("2352.73", false)]
22578 #[case::price_restatement("2352.74", true)]
22579 fn cached_parent_fill_is_not_deferred_during_child_recovery(
22580 #[case] price: &str,
22581 #[case] deferred: bool,
22582 ) {
22583 let (client, cache, mut rx) = create_execution_client();
22584 let instrument_id = register_test_instrument(&client, &cache);
22585 let instrument = cache.borrow().instrument(&instrument_id).unwrap().clone();
22586 LIGHTER_INSTRUMENT_CACHE.insert(instrument_id, instrument.clone());
22587 let parent_id = ClientOrderId::new("HTTP-CACHED-PARENT");
22588 let parent_venue = VenueOrderId::new("710001");
22589 let child_id = ClientOrderId::new("HTTP-RECOVERING-CHILD");
22590 let child_venue = VenueOrderId::new("710002");
22591 let parent = OrderTestBuilder::new(OrderType::Limit)
22592 .trader_id(trader_id())
22593 .strategy_id(strategy_id())
22594 .instrument_id(instrument_id)
22595 .client_order_id(parent_id)
22596 .side(OrderSide::Buy)
22597 .quantity(Quantity::from("0.1336"))
22598 .price(Price::from("2352.73"))
22599 .build();
22600 cache_accepted_order(&cache, parent, parent_venue, Some(client_id()));
22601 let parent = cache.borrow().order_owned(&parent_id).unwrap();
22602 let mut report =
22603 reconciliation_fill_report(instrument_id, 101, parent_venue, "CACHED-PARENT-FILL");
22604 let opening = TestOrderEventStubs::filled(
22605 &parent,
22606 &instrument,
22607 Some(report.trade_id),
22608 None,
22609 Some(report.last_px),
22610 Some(report.last_qty),
22611 Some(report.liquidity_side),
22612 Some(report.commission),
22613 Some(report.ts_event),
22614 Some(account_id()),
22615 );
22616 cache.borrow_mut().update_order(&opening).unwrap();
22617 let child = OrderTestBuilder::new(OrderType::StopLimit)
22618 .trader_id(trader_id())
22619 .strategy_id(strategy_id())
22620 .instrument_id(instrument_id)
22621 .client_order_id(child_id)
22622 .side(OrderSide::Sell)
22623 .quantity(Quantity::from("0.1336"))
22624 .price(Price::from("2301.00"))
22625 .trigger_price(Price::from("2302.00"))
22626 .parent_order_id(parent_id)
22627 .reduce_only(true)
22628 .build();
22629 cache_accepted_order(&cache, child, child_venue, Some(client_id()));
22630 let child = cache.borrow().order_owned(&child_id).unwrap();
22631 client
22632 .dispatch
22633 .restore_reconciled_order(&child, 102, child_venue, false)
22634 .unwrap();
22635 let identity = client.dispatch.order_identity(&child_id).unwrap();
22636 let mut child_report =
22637 reconciliation_fill_report(instrument_id, 102, child_venue, "NEW-CHILD-FILL");
22638 child_report.order_side = OrderSide::Sell;
22639 child_report.last_qty = Quantity::from("0.0444");
22640 child_report.last_px = Price::from("2301.23");
22641 let child_fill = child_fill_from_report(&child_report, &identity, trader_id()).unwrap();
22642 client.dispatch.defer_fill(
22643 client
22644 .dispatch
22645 .order_events
22646 .lock()
22647 .entry(child_id)
22648 .or_default(),
22649 child_fill.clone(),
22650 );
22651 report.last_px = Price::from(price);
22652
22653 let sweep = finish_fill_reports(
22654 &client.core,
22655 &client.dispatch,
22656 BTreeSet::new(),
22657 Ok(FillSweep {
22658 reports: vec![report.clone()],
22659 covers_window: true,
22660 withheld_instruments: AHashSet::new(),
22661 }),
22662 )
22663 .unwrap();
22664
22665 let events = client.dispatch.order_events.lock();
22666 assert!(sweep.reports.is_empty());
22667 assert!(!sweep.covers_window);
22668 assert_eq!(
22669 sweep.withheld_instruments,
22670 AHashSet::from_iter([instrument_id])
22671 );
22672 assert_eq!(events.get(&parent_id).is_some(), deferred);
22673 assert_eq!(events[&child_id].fills, vec![child_fill]);
22674
22675 if deferred {
22676 let fills = &events[&parent_id].fills;
22677 assert_eq!(fills.len(), 1);
22678 assert_eq!(fills[0].trade_id, report.trade_id);
22679 assert_eq!(fills[0].client_order_id, parent_id);
22680 assert_eq!(fills[0].venue_order_id, parent_venue);
22681 assert_eq!(fills[0].last_qty, report.last_qty);
22682 assert_eq!(fills[0].last_px, report.last_px);
22683 assert_eq!(fills[0].commission, Some(report.commission));
22684 }
22685
22686 assert_eq!(
22687 cache.borrow().order(&parent_id).unwrap().trade_ids(),
22688 vec![&report.trade_id]
22689 );
22690 assert!(drain_events(&mut rx).is_empty());
22691 }
22692
22693 #[rstest]
22694 #[case::immediate(false, false, 1)]
22695 #[case::deferred(false, true, 1)]
22696 #[case::repeated(false, false, 16)]
22697 #[case::deferred_repeated(false, true, 16)]
22698 #[case::first_frame(true, false, 1)]
22699 fn self_trade_reduce_updates_cache_and_reports(
22700 #[case] first_frame: bool,
22701 #[case] deferred: bool,
22702 #[case] repeats: usize,
22703 ) {
22704 let (mut rig, cache, mut engine, mut frame) = reduce_order_replay();
22705 let mut events = Vec::new();
22706
22707 if !first_frame {
22708 dispatch_reduce_frame(&rig, &frame, false);
22709 if !deferred {
22710 events.extend(consume_reduce_events(&mut rig, &mut engine));
22711 }
22712 }
22713
22714 frame.remaining_base_amount = Decimal::from_str("0.0001").unwrap();
22715 frame.updated_at += 1_000;
22716 for _ in 0..repeats {
22717 dispatch_reduce_frame(&rig, &frame, false);
22718 if !deferred {
22719 events.extend(consume_reduce_events(&mut rig, &mut engine));
22720 }
22721 }
22722
22723 events.extend(consume_reduce_events(&mut rig, &mut engine));
22724 let order = cache.borrow().order(&rig.cloid).unwrap().clone();
22725 let report = rig.dispatch.translate_order_cloid(
22726 parse_http_order_to_report(
22727 &frame,
22728 &rig.registry,
22729 account_id(),
22730 UnixNanos::from(5),
22731 &rig.dispatch,
22732 )
22733 .unwrap(),
22734 );
22735 engine.reconcile_order_status_report(&report);
22736 engine.reconcile_order_status_report(&report);
22737 let reconciled = cache.borrow().order(&rig.cloid).unwrap().clone();
22738 rig.dispatch = WsDispatchState::new();
22739 rig.dispatch
22740 .seed_order_restorations(vec![reconciled.clone()]);
22741 dispatch_reduce_frame(&rig, &frame, false);
22742 let replayed = consume_reduce_events(&mut rig, &mut engine);
22743
22744 assert_eq!(events.len(), 2);
22745 assert!(matches!(events[0], OrderEventAny::Accepted(_)));
22746
22747 let OrderEventAny::Updated(update) = &events[1] else {
22748 panic!("expected quantity update");
22749 };
22750
22751 assert_eq!(update.quantity, Quantity::from("0.0001"));
22752 assert!(update.reconciliation);
22753 assert_eq!(order.status(), OrderStatus::Accepted);
22754 assert_eq!(order.quantity(), Quantity::from("0.0001"));
22755 assert_eq!(order.leaves_qty(), Quantity::from("0.0001"));
22756 assert_eq!(order.filled_qty(), Quantity::from("0.0000"));
22757 assert_eq!(report.quantity, Quantity::from("0.0001"));
22758 assert_eq!(report.filled_qty, Quantity::from("0.0000"));
22759 assert_eq!(reconciled, order);
22760 assert!(replayed.is_empty());
22761 }
22762
22763 #[rstest]
22764 #[case::frame_first(false, false, false)]
22765 #[case::trade_first(true, false, false)]
22766 #[case::reduced_then_frame(false, true, false)]
22767 #[case::reduced_then_trade(true, true, false)]
22768 #[case::deferred_frame(false, false, true)]
22769 #[case::deferred_trade(true, false, true)]
22770 fn self_trade_reduce_final_fill_ordering(
22771 #[case] trade_first: bool,
22772 #[case] reduced_open: bool,
22773 #[case] deferred: bool,
22774 #[values(false, true)] first_frame: bool,
22775 ) {
22776 let (mut rig, cache, mut engine, mut frame) = reduce_order_replay();
22777 if !first_frame {
22778 dispatch_reduce_frame(&rig, &frame, false);
22779 consume_reduce_events(&mut rig, &mut engine);
22780 }
22781
22782 frame.remaining_base_amount = Decimal::from_str("0.0001").unwrap();
22783 frame.updated_at += 1_000;
22784 if reduced_open {
22785 dispatch_reduce_frame(&rig, &frame, false);
22786 consume_reduce_events(&mut rig, &mut engine);
22787 }
22788
22789 let mut trade = dispatcher_test_trade(&rig, true);
22790 trade.bid_id = frame.order_index;
22791 trade.bid_id_str = Some(frame.order_id.clone());
22792 trade.size = Decimal::from_str("0.0001").unwrap();
22793 trade.usd_amount = trade.size * trade.price;
22794 frame.status = LighterOrderStatus::Filled;
22795 frame.remaining_base_amount = Decimal::ZERO;
22796 frame.filled_base_amount = trade.size;
22797 frame.filled_quote_amount = trade.usd_amount;
22798 frame.updated_at += 1_000;
22799
22800 if trade_first {
22801 dispatch_lighter_trade(
22802 &trade,
22803 &rig.dispatch,
22804 &rig.emitter,
22805 &rig.registry,
22806 account_id(),
22807 trader_id(),
22808 Some(TEST_ACCOUNT_INDEX_I64),
22809 UnixNanos::from(3),
22810 );
22811 consume_reduce_events(&mut rig, &mut engine);
22812 }
22813
22814 dispatch_reduce_frame(&rig, &frame, deferred);
22815 consume_reduce_events(&mut rig, &mut engine);
22816
22817 if !trade_first {
22818 dispatch_lighter_trade(
22819 &trade,
22820 &rig.dispatch,
22821 &rig.emitter,
22822 &rig.registry,
22823 account_id(),
22824 trader_id(),
22825 Some(TEST_ACCOUNT_INDEX_I64),
22826 UnixNanos::from(4),
22827 );
22828 consume_reduce_events(&mut rig, &mut engine);
22829 }
22830
22831 let order = cache.borrow().order(&rig.cloid).unwrap().clone();
22832
22833 assert_eq!(order.quantity(), Quantity::from("0.0001"));
22834 assert_eq!(order.filled_qty(), Quantity::from("0.0001"));
22835 assert_eq!(order.leaves_qty(), Quantity::from("0.0000"));
22836 assert_eq!(order.status(), OrderStatus::Filled);
22837 assert_eq!(
22838 order.trade_ids(),
22839 vec![&parse_lighter_trade_id(&trade).unwrap()]
22840 );
22841 }
22842
22843 #[rstest]
22844 #[case::stream(false)]
22845 #[case::report(true)]
22846 fn self_trade_reduce_cancellation_remains_canceled(#[case] report: bool) {
22847 let (mut rig, cache, mut engine, mut frame) = reduce_order_replay();
22848 dispatch_reduce_frame(&rig, &frame, false);
22849 consume_reduce_events(&mut rig, &mut engine);
22850 frame.remaining_base_amount = Decimal::from_str("0.0001").unwrap();
22851 frame.updated_at += 1_000;
22852 dispatch_reduce_frame(&rig, &frame, false);
22853 consume_reduce_events(&mut rig, &mut engine);
22854 frame.status = LighterOrderStatus::Canceled;
22855 frame.remaining_base_amount = Decimal::ZERO;
22856 frame.updated_at += 1_000;
22857
22858 if report {
22859 let report = rig.dispatch.translate_order_cloid(
22860 parse_http_order_to_report(
22861 &frame,
22862 &rig.registry,
22863 account_id(),
22864 UnixNanos::from(5),
22865 &rig.dispatch,
22866 )
22867 .unwrap(),
22868 );
22869 engine.reconcile_order_status_report(&report);
22870 } else {
22871 dispatch_reduce_frame(&rig, &frame, false);
22872 consume_reduce_events(&mut rig, &mut engine);
22873 }
22874
22875 let order = cache.borrow().order(&rig.cloid).unwrap().clone();
22876
22877 assert_eq!(order.status(), OrderStatus::Canceled);
22878 assert_eq!(order.filled_qty(), Quantity::from("0.0000"));
22879 assert!(order.trade_ids().is_empty());
22880 assert_eq!(
22881 order.quantity(),
22882 Quantity::from(if report { "0.0027" } else { "0.0001" })
22883 );
22884 }
22885
22886 fn reduce_order_replay() -> (
22887 DispatcherRig,
22888 Rc<RefCell<Cache>>,
22889 ExecutionEngine,
22890 LighterOrder,
22891 ) {
22892 let rig = dispatcher_rig("REDUCE-REPLAY");
22893 let mut frame = dispatcher_test_order(&rig, LighterOrderStatus::Open);
22894 frame.initial_base_amount = Decimal::from_str("0.0027").unwrap();
22895 frame.remaining_base_amount = frame.initial_base_amount;
22896 let order = OrderTestBuilder::new(OrderType::Limit)
22897 .trader_id(trader_id())
22898 .strategy_id(strategy_id())
22899 .instrument_id(rig.instrument_id)
22900 .client_order_id(rig.cloid)
22901 .side(OrderSide::Buy)
22902 .quantity(Quantity::from("0.0027"))
22903 .price(Price::from("2352.74"))
22904 .build();
22905 rig.dispatch.register_create_identity(&order).unwrap();
22906 rig.dispatch
22907 .mark_order_submission(&rig.cloid, TEST_SUBMISSION_NONCE);
22908 let cache = Rc::new(RefCell::new(Cache::default()));
22909 cache
22910 .borrow_mut()
22911 .add_instrument(
22912 LIGHTER_INSTRUMENT_CACHE
22913 .get(&rig.instrument_id)
22914 .unwrap()
22915 .value()
22916 .clone(),
22917 )
22918 .unwrap();
22919 cache
22920 .borrow_mut()
22921 .add_order(order, None, Some(client_id()), false)
22922 .unwrap();
22923 let clock: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(VirtualClock::default()));
22924 let mut engine = ExecutionEngine::new(clock, cache.clone(), None);
22925 engine.process(&OrderEventAny::Submitted(OrderSubmitted::new(
22926 trader_id(),
22927 strategy_id(),
22928 rig.instrument_id,
22929 rig.cloid,
22930 account_id(),
22931 UUID4::new(),
22932 UnixNanos::from(1),
22933 UnixNanos::from(1),
22934 )));
22935 (rig, cache, engine, frame)
22936 }
22937
22938 fn dispatch_reduce_frame(rig: &DispatcherRig, frame: &LighterOrder, deferred: bool) {
22939 if deferred {
22940 dispatch_deferred_order(rig.cloid, frame, &rig.dispatch, &rig.emitter);
22941 } else {
22942 dispatch_lighter_order(
22943 frame,
22944 &rig.dispatch,
22945 &rig.emitter,
22946 &rig.registry,
22947 account_id(),
22948 trader_id(),
22949 UnixNanos::from(3),
22950 );
22951 }
22952 }
22953
22954 fn consume_reduce_events(
22955 rig: &mut DispatcherRig,
22956 engine: &mut ExecutionEngine,
22957 ) -> Vec<OrderEventAny> {
22958 let mut orders = Vec::new();
22959
22960 for event in drain_events(&mut rig.rx) {
22961 match event {
22962 ExecutionEvent::Order(event) => {
22963 engine.process(&event);
22964 orders.push(event);
22965 }
22966 ExecutionEvent::Report(report) => engine.reconcile_execution_report(&report),
22967 other => panic!("unexpected event: {other:?}"),
22968 }
22969 }
22970
22971 orders
22972 }
22973
22974 #[rstest]
22975 #[case::ioc(LighterOrderStatus::CanceledNotEnoughLiquidity, false)]
22976 #[case::expiry(LighterOrderStatus::CanceledExpired, false)]
22977 #[case::post_only(LighterOrderStatus::CanceledPostOnly, false)]
22978 #[case::resting(LighterOrderStatus::Canceled, true)]
22979 fn terminal_waits_for_all_distinct_fills(
22980 #[case] status: LighterOrderStatus,
22981 #[case] trade_first: bool,
22982 ) {
22983 let mut rig = dispatcher_rig("TERMINAL-FILLS");
22984 register_identity(&rig);
22985 rig.dispatch.mark_accepted_emitted(rig.cloid);
22986 let identity = rig.dispatch.order_identity(&rig.cloid).unwrap();
22987 let mut order = dispatcher_test_order(&rig, status);
22988 order.filled_base_amount = Decimal::from_str("0.0030").unwrap();
22989 let mut first = dispatcher_test_trade(&rig, true);
22990 first.bid_id = order.order_index;
22991 first.bid_id_str = Some(order.order_id.clone());
22992 first.size = Decimal::from_str("0.0010").unwrap();
22993 let mut second = first.clone();
22994 second.trade_id += 1;
22995 second.trade_id_str = Some(second.trade_id.to_string());
22996 second.size = Decimal::from_str("0.0020").unwrap();
22997
22998 let send_trade = |trade: &LighterTrade| {
22999 dispatch_lighter_trade(
23000 trade,
23001 &rig.dispatch,
23002 &rig.emitter,
23003 &rig.registry,
23004 account_id(),
23005 trader_id(),
23006 Some(TEST_ACCOUNT_INDEX_I64),
23007 UnixNanos::from(2),
23008 );
23009 };
23010
23011 if trade_first {
23012 send_trade(&first);
23013 }
23014
23015 dispatch_lighter_order(
23016 &order,
23017 &rig.dispatch,
23018 &rig.emitter,
23019 &rig.registry,
23020 account_id(),
23021 trader_id(),
23022 UnixNanos::from(1),
23023 );
23024 let deadline = rig.dispatch.terminals_pending.lock()[&rig.cloid].retry_at;
23025 dispatch_lighter_order(
23026 &order,
23027 &rig.dispatch,
23028 &rig.emitter,
23029 &rig.registry,
23030 account_id(),
23031 trader_id(),
23032 UnixNanos::from(3),
23033 );
23034 assert_eq!(
23035 rig.dispatch.terminals_pending.lock()[&rig.cloid].retry_at,
23036 deadline
23037 );
23038 send_trade(&first);
23039 send_trade(&first);
23040 assert_eq!(identity.filled_qty(), Decimal::from_str("0.0010").unwrap());
23041 assert!(rig.dispatch.order_identities.contains_key(&rig.cloid));
23042 let mut events = drain_events(&mut rig.rx);
23043 assert_eq!(events.len(), 1);
23044 assert!(
23045 matches!(&events[0], ExecutionEvent::Order(OrderEventAny::Filled(fill))
23046 if fill.trade_id.as_str() == first.trade_id_str.as_deref().unwrap()
23047 && fill.last_qty == Quantity::from("0.0010"))
23048 );
23049 dispatch_lighter_trade(
23050 &second,
23051 &rig.dispatch,
23052 &rig.emitter,
23053 &rig.registry,
23054 account_id(),
23055 trader_id(),
23056 Some(TEST_ACCOUNT_INDEX_I64),
23057 UnixNanos::from(4),
23058 );
23059 events.extend(drain_events(&mut rig.rx));
23060 assert_eq!(events.len(), 3);
23061 assert!(
23062 matches!(&events[1], ExecutionEvent::Order(OrderEventAny::Filled(fill))
23063 if fill.last_qty == Quantity::from("0.0020"))
23064 );
23065
23066 let terminal_ts = match &events[2] {
23067 ExecutionEvent::Order(OrderEventAny::Canceled(event)) => {
23068 assert_ne!(status, LighterOrderStatus::CanceledExpired);
23069 event.ts_event
23070 }
23071 ExecutionEvent::Order(OrderEventAny::Expired(event)) => {
23072 assert_eq!(status, LighterOrderStatus::CanceledExpired);
23073 event.ts_event
23074 }
23075 event => panic!("expected terminal after fills, was {event:?}"),
23076 };
23077
23078 assert_eq!(terminal_ts, UnixNanos::from(1_777_941_383_900_000_000));
23079 assert_eq!(identity.filled_qty(), Decimal::from_str("0.0030").unwrap());
23080 assert!(rig.dispatch.terminals_pending.lock().is_empty());
23081 assert!(!rig.dispatch.order_identities.contains_key(&rig.cloid));
23082 }
23083
23084 #[rstest]
23085 fn terminal_after_fills_has_no_delay() {
23086 let mut rig = dispatcher_rig("TERMINAL-COVERED");
23087 register_identity(&rig);
23088 let mut order = dispatcher_test_order(&rig, LighterOrderStatus::Canceled);
23089 let mut trade = dispatcher_test_trade(&rig, true);
23090 trade.bid_id = order.order_index;
23091 trade.bid_id_str = Some(order.order_id.clone());
23092 order.filled_base_amount = trade.size;
23093 dispatch_lighter_trade(
23094 &trade,
23095 &rig.dispatch,
23096 &rig.emitter,
23097 &rig.registry,
23098 account_id(),
23099 trader_id(),
23100 Some(TEST_ACCOUNT_INDEX_I64),
23101 UnixNanos::from(1),
23102 );
23103 dispatch_lighter_order(
23104 &order,
23105 &rig.dispatch,
23106 &rig.emitter,
23107 &rig.registry,
23108 account_id(),
23109 trader_id(),
23110 UnixNanos::from(2),
23111 );
23112 let events = drain_events(&mut rig.rx);
23113 assert_eq!(events.len(), 3);
23114 assert!(matches!(
23115 &events[0],
23116 ExecutionEvent::Order(OrderEventAny::Accepted(_))
23117 ));
23118 assert!(matches!(
23119 &events[1],
23120 ExecutionEvent::Order(OrderEventAny::Filled(_))
23121 ));
23122 assert!(matches!(
23123 &events[2],
23124 ExecutionEvent::Order(OrderEventAny::Canceled(_))
23125 ));
23126 assert!(rig.dispatch.terminals_pending.lock().is_empty());
23127 }
23128
23129 #[rstest]
23130 #[case::modify_hold(true)]
23131 #[case::fill_hold(false)]
23132 #[tokio::test]
23133 async fn terminal_preserves_later_holds_and_emits_once(#[case] modify_pending: bool) {
23134 let mut rig = dispatcher_rig("TERMINAL-MODIFY");
23135 register_identity(&rig);
23136 rig.dispatch.mark_accepted_emitted(rig.cloid);
23137 let mut order = dispatcher_test_order(&rig, LighterOrderStatus::Canceled);
23138 let mut trade = dispatcher_test_trade(&rig, true);
23139 trade.bid_id = order.order_index;
23140 trade.bid_id_str = Some(order.order_id.clone());
23141 order.filled_base_amount = trade.size;
23142 dispatch_lighter_order(
23143 &order,
23144 &rig.dispatch,
23145 &rig.emitter,
23146 &rig.registry,
23147 account_id(),
23148 trader_id(),
23149 UnixNanos::from(1),
23150 );
23151
23152 let command = ModifyOrder::new(
23153 trader_id(),
23154 Some(client_id()),
23155 strategy_id(),
23156 rig.instrument_id,
23157 rig.cloid,
23158 Some(VenueOrderId::new(&order.order_id)),
23159 None,
23160 Some(Price::from("2350.00")),
23161 None,
23162 UUID4::new(),
23163 UnixNanos::from(2),
23164 None,
23165 None,
23166 );
23167 assert!(rig.dispatch.set_pending_modify_action(&command));
23168 dispatch_lighter_order(
23169 &order,
23170 &rig.dispatch,
23171 &rig.emitter,
23172 &rig.registry,
23173 account_id(),
23174 trader_id(),
23175 UnixNanos::from(3),
23176 );
23177 dispatch_lighter_trade(
23178 &trade,
23179 &rig.dispatch,
23180 &rig.emitter,
23181 &rig.registry,
23182 account_id(),
23183 trader_id(),
23184 Some(TEST_ACCOUNT_INDEX_I64),
23185 UnixNanos::from(4),
23186 );
23187
23188 if !modify_pending {
23189 assert!(
23190 rig.dispatch
23191 .clear_pending_modify_if_command(&rig.cloid, command.command_id)
23192 );
23193 }
23194
23195 let (http_client, hits, server) = terminal_trades_server(&order, Vec::new()).await;
23196 let (client, _, _) = create_execution_client();
23197 let mut context = acked_probe_context(&client);
23198 context.http_client = http_client;
23199 context.dispatch = rig.dispatch.clone();
23200 context.registry = rig.registry.clone();
23201 context.emitter = rig.emitter.clone();
23202 for _ in 0..=ACKED_ORDER_PROBE_ATTEMPTS {
23203 recover_terminals(&context, true).await;
23204 }
23205
23206 assert_eq!(hits.load(Ordering::Relaxed), ACKED_ORDER_PROBE_ATTEMPTS);
23207 {
23208 let mut events = rig.dispatch.order_events.lock();
23209 release_terminal(rig.cloid, true, &rig.dispatch, &rig.emitter, &events);
23210 assert!(drain_events(&mut rig.rx).is_empty());
23211 assert!(
23212 rig.dispatch
23213 .terminals_pending
23214 .lock()
23215 .contains_key(&rig.cloid)
23216 );
23217 assert!(rig.dispatch.order_identities.contains_key(&rig.cloid));
23218
23219 if modify_pending {
23220 assert!(
23221 rig.dispatch
23222 .clear_pending_modify_if_command(&rig.cloid, command.command_id)
23223 );
23224 }
23225
23226 let deferred = events.remove(&rig.cloid).unwrap();
23227 flush_modify_events(
23228 rig.cloid,
23229 deferred,
23230 &rig.dispatch,
23231 &rig.emitter,
23232 &mut events,
23233 );
23234 }
23235
23236 let events = drain_events(&mut rig.rx);
23237 assert_eq!(events.len(), 2);
23238 assert!(
23239 matches!(&events[0], ExecutionEvent::Order(OrderEventAny::Filled(fill)) if fill.trade_id == parse_lighter_trade_id(&trade).unwrap())
23240 );
23241 assert!(matches!(
23242 &events[1],
23243 ExecutionEvent::Order(OrderEventAny::Canceled(_))
23244 ));
23245 assert!(rig.dispatch.terminals_pending.lock().is_empty());
23246 assert!(rig.dispatch.order_events.lock().is_empty());
23247 assert!(!rig.dispatch.order_identities.contains_key(&rig.cloid));
23248 server.abort();
23249 let _ = server.await;
23250 }
23251
23252 async fn terminal_trades_server(
23253 order: &LighterOrder,
23254 trades: Vec<LighterTrade>,
23255 ) -> (
23256 LighterHttpClient,
23257 Arc<AtomicUsize>,
23258 tokio::task::JoinHandle<()>,
23259 ) {
23260 let hits = Arc::new(AtomicUsize::new(0));
23261 let count = hits.clone();
23262 let order_index = order.order_index.to_string();
23263 let market_index = order.market_index.to_string();
23264
23265 let raw = order.clone();
23266
23267 let app = Router::new()
23268 .route(
23269 "/api/v1/accountActiveOrders",
23270 get(|| async { axum::Json(serde_json::json!({"code":200,"orders":[]})) }),
23271 )
23272 .route(
23273 "/api/v1/accountInactiveOrders",
23274 get(move || {
23275 let raw = raw.clone();
23276 async move { axum::Json(serde_json::json!({"code":200,"orders":[raw]})) }
23277 }),
23278 )
23279 .route(
23280 "/api/v1/trades",
23281 get(
23282 move |axum::extract::Query(query): axum::extract::Query<
23283 BTreeMap<String, String>,
23284 >| {
23285 let trades = trades.clone();
23286 let count = count.clone();
23287 let order_index = order_index.clone();
23288 let market_index = market_index.clone();
23289 async move {
23290 count.fetch_add(1, Ordering::Relaxed);
23291 assert_eq!(query.get("order_index"), Some(&order_index));
23292 assert_eq!(query.get("market_id"), Some(&market_index));
23293 assert_eq!(
23294 query.get("account_index"),
23295 Some(&TEST_ACCOUNT_INDEX_I64.to_string())
23296 );
23297 assert_eq!(query.get("sort_by").map(String::as_str), Some("timestamp"));
23298 assert_eq!(query.get("sort_dir").map(String::as_str), Some("desc"));
23299 assert_eq!(query.get("limit").map(String::as_str), Some("100"));
23300 assert!(!query.contains_key("from"));
23301 assert!(query.get("auth").is_some_and(|auth| !auth.is_empty()));
23302 axum::Json(serde_json::json!({"code":200,"trades":trades}))
23303 }
23304 },
23305 ),
23306 );
23307
23308 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
23309 let addr = listener.local_addr().unwrap();
23310 let server = tokio::spawn(async move {
23311 axum::serve(listener, app).await.unwrap();
23312 });
23313
23314 let http_client = LighterHttpClient::new(
23315 LighterEnvironment::Testnet,
23316 Some(format!("http://{addr}")),
23317 1,
23318 None,
23319 )
23320 .unwrap();
23321 (http_client, hits, server)
23322 }
23323
23324 #[rstest]
23325 #[case::recovered(true, false)]
23326 #[case::exhausted(false, false)]
23327 #[case::foreign_order(false, true)]
23328 #[tokio::test]
23329 async fn terminal_recovery_is_order_scoped_and_bounded(
23330 #[case] recovered: bool,
23331 #[case] foreign_order: bool,
23332 ) {
23333 let mut rig = dispatcher_rig("TERMINAL-RECOVERY");
23334 register_identity(&rig);
23335 rig.dispatch.mark_accepted_emitted(rig.cloid);
23336 let mut order = dispatcher_test_order(&rig, LighterOrderStatus::CanceledNotEnoughLiquidity);
23337 let mut trade = dispatcher_test_trade(&rig, true);
23338 trade.bid_id = order.order_index;
23339 trade.bid_id_str = Some(order.order_id.clone());
23340 trade.size = Decimal::from_str("0.0025").unwrap();
23341 order.filled_base_amount = trade.size;
23342 dispatch_lighter_order(
23343 &order,
23344 &rig.dispatch,
23345 &rig.emitter,
23346 &rig.registry,
23347 account_id(),
23348 trader_id(),
23349 UnixNanos::from(1),
23350 );
23351
23352 let trades = if foreign_order {
23353 trade.bid_id += 1;
23354 trade.bid_id_str = Some(trade.bid_id.to_string());
23355 vec![trade]
23356 } else if recovered {
23357 vec![trade.clone(), trade]
23358 } else {
23359 Vec::new()
23360 };
23361
23362 let (http_client, hits, server) = terminal_trades_server(&order, trades).await;
23363 let (client, _, _) = create_execution_client();
23364 let mut context = acked_probe_context(&client);
23365 context.http_client = http_client;
23366 context.dispatch = rig.dispatch.clone();
23367 context.registry = rig.registry.clone();
23368 context.emitter = rig.emitter.clone();
23369 recover_terminals(&context, false).await;
23370 assert_eq!(hits.load(Ordering::Relaxed), 0, "deadline must be honored");
23371
23372 if foreign_order {
23373 let error = recover_terminal_trades(order.market_index, order.order_index, &context)
23374 .await
23375 .unwrap_err();
23376 assert_eq!(
23377 error.to_string(),
23378 "Lighter terminal recovery returned a trade outside the requested order"
23379 );
23380 hits.store(0, Ordering::Relaxed);
23381 }
23382
23383 let attempts = if recovered {
23384 1
23385 } else {
23386 ACKED_ORDER_PROBE_ATTEMPTS
23387 };
23388
23389 for attempt in 1..=attempts {
23390 recover_terminals(&context, true).await;
23391 assert_eq!(hits.load(Ordering::Relaxed), attempt);
23392
23393 if attempt < attempts {
23394 assert!(drain_events(&mut rig.rx).is_empty());
23395 assert_eq!(
23396 rig.dispatch.terminals_pending.lock()[&rig.cloid].attempts,
23397 attempt
23398 );
23399 }
23400 }
23401
23402 let events = drain_events(&mut rig.rx);
23403 assert_eq!(events.len(), if recovered { 2 } else { 1 });
23404
23405 if recovered {
23406 assert!(
23407 matches!(&events[0], ExecutionEvent::Order(OrderEventAny::Filled(fill))
23408 if fill.last_qty == Quantity::from("0.0025"))
23409 );
23410 }
23411
23412 assert!(matches!(
23413 events.last(),
23414 Some(ExecutionEvent::Order(OrderEventAny::Canceled(_)))
23415 ));
23416 assert!(rig.dispatch.terminals_pending.lock().is_empty());
23417 recover_terminals(&context, true).await;
23418 assert!(drain_events(&mut rig.rx).is_empty());
23419 assert_eq!(hits.load(Ordering::Relaxed), attempts);
23420 server.abort();
23421 let _ = server.await;
23422 }
23423
23424 #[rstest]
23425 #[case::reconnect(false)]
23426 #[case::stop(true)]
23427 #[tokio::test]
23428 async fn terminal_worker_recovers_immediately_on_lifecycle(#[case] stop: bool) {
23429 let mut rig = dispatcher_rig("TERMINAL-LIFECYCLE");
23430 register_identity(&rig);
23431 rig.dispatch.mark_accepted_emitted(rig.cloid);
23432 let mut order = dispatcher_test_order(&rig, LighterOrderStatus::CanceledExpired);
23433 let mut trade = dispatcher_test_trade(&rig, true);
23434 trade.bid_id = order.order_index;
23435 trade.bid_id_str = Some(order.order_id.clone());
23436 trade.size = Decimal::from_str("0.0025").unwrap();
23437 order.filled_base_amount = trade.size;
23438 dispatch_lighter_order(
23439 &order,
23440 &rig.dispatch,
23441 &rig.emitter,
23442 &rig.registry,
23443 account_id(),
23444 trader_id(),
23445 UnixNanos::from(1),
23446 );
23447 rig.dispatch
23448 .terminals_pending
23449 .lock()
23450 .get_mut(&rig.cloid)
23451 .unwrap()
23452 .retry_at = tokio::time::Instant::now() + Duration::from_secs(3600);
23453 let (http_client, hits, server) = terminal_trades_server(&order, vec![trade]).await;
23454 let (mut client, _, _) = create_execution_client();
23455 client.http_client = http_client;
23456 client.dispatch = rig.dispatch.clone();
23457 client.registry = rig.registry.clone();
23458 client.emitter = rig.emitter.clone();
23459 let context = acked_probe_context(&client);
23460 client
23461 .terminal_recovery_handle
23462 .spawn(run_terminal_recovery(
23463 context,
23464 client.terminal_producers_stopped.clone(),
23465 ))
23466 .unwrap();
23467
23468 if stop {
23469 client.stop().unwrap();
23470 } else {
23471 client.dispatch.terminals_recover.notify_one();
23472 }
23473
23474 if !stop {
23475 wait_until_async(
23476 || async { rig.dispatch.terminals_pending.lock().is_empty() },
23477 Duration::from_secs(2),
23478 )
23479 .await;
23480 client.stop().unwrap();
23481 }
23482
23483 client.finish_session_shutdown().await.unwrap();
23484 let events = drain_events(&mut rig.rx);
23485 assert_eq!(hits.load(Ordering::Relaxed), 1);
23486 assert_eq!(events.len(), 2);
23487 assert!(
23488 matches!(&events[0], ExecutionEvent::Order(OrderEventAny::Filled(fill))
23489 if fill.last_qty == Quantity::from("0.0025"))
23490 );
23491 assert!(
23492 matches!(&events[1], ExecutionEvent::Order(OrderEventAny::Expired(event))
23493 if event.ts_event == UnixNanos::from(1_777_941_383_900_000_000))
23494 );
23495 assert!(client.terminal_recovery_handle.is_none());
23496 assert!(rig.dispatch.terminals_pending.lock().is_empty());
23497 server.abort();
23498 let _ = server.await;
23499 }
23500
23501 #[rstest]
23502 #[case::restored(true, false)]
23503 #[case::reconciled(false, false)]
23504 #[case::declined(false, true)]
23505 fn terminal_preserves_applied_fills_and_post_only_state(
23506 #[case] restored: bool,
23507 #[case] declined: bool,
23508 ) {
23509 let mut rig = dispatcher_rig("TERMINAL-BASELINE");
23510 register_identity(&rig);
23511 let mut frame = dispatcher_test_order(&rig, LighterOrderStatus::Open);
23512 let instrument = LIGHTER_INSTRUMENT_CACHE
23513 .get(&rig.instrument_id)
23514 .unwrap()
23515 .value()
23516 .clone();
23517 let mut order = OrderTestBuilder::new(OrderType::Limit)
23518 .trader_id(trader_id())
23519 .strategy_id(strategy_id())
23520 .instrument_id(rig.instrument_id)
23521 .client_order_id(rig.cloid)
23522 .side(OrderSide::Buy)
23523 .quantity(Quantity::from("0.0050"))
23524 .price(Price::from("2352.74"))
23525 .post_only(true)
23526 .build();
23527 dispatch_lighter_order(
23528 &frame,
23529 &rig.dispatch,
23530 &rig.emitter,
23531 &rig.registry,
23532 account_id(),
23533 trader_id(),
23534 UnixNanos::from(1),
23535 );
23536 frame.price = Decimal::from_str("2353.00").unwrap();
23537 dispatch_lighter_order(
23538 &frame,
23539 &rig.dispatch,
23540 &rig.emitter,
23541 &rig.registry,
23542 account_id(),
23543 trader_id(),
23544 UnixNanos::from(2),
23545 );
23546
23547 for event in drain_events(&mut rig.rx) {
23548 let ExecutionEvent::Order(event) = event else {
23549 panic!("expected order event");
23550 };
23551
23552 order.apply(event).unwrap();
23553 }
23554
23555 let mut first = dispatcher_test_trade(&rig, true);
23556 first.bid_id = frame.order_index;
23557 first.bid_id_str = Some(frame.order_id.clone());
23558 first.size = Decimal::from_str("0.0010").unwrap();
23559 let fill = parse_lighter_order_filled(
23560 &first,
23561 &instrument,
23562 &rig.dispatch.order_identity(&rig.cloid).unwrap(),
23563 rig.cloid,
23564 account_id(),
23565 trader_id(),
23566 TEST_ACCOUNT_INDEX_I64,
23567 UnixNanos::from(3),
23568 )
23569 .unwrap()
23570 .unwrap();
23571 let (mut client, cache, _) = create_execution_client();
23572 client.dispatch = rig.dispatch.clone();
23573 client.emitter = rig.emitter.clone();
23574 client.subscribe_applied_fills();
23575 cache
23576 .borrow_mut()
23577 .add_order(order.clone(), None, Some(client_id()), false)
23578 .unwrap();
23579
23580 if !restored {
23581 let mut report = parse_ws_fill_report(
23582 &first,
23583 TEST_ACCOUNT_INDEX_I64,
23584 &instrument,
23585 account_id(),
23586 UnixNanos::from(3),
23587 )
23588 .unwrap()
23589 .unwrap();
23590 report.client_order_id = Some(rig.cloid);
23591
23592 let sweep = finish_fill_reports(
23593 &client.core,
23594 &rig.dispatch,
23595 BTreeSet::new(),
23596 Ok(FillSweep {
23597 reports: vec![report.clone()],
23598 covers_window: true,
23599 withheld_instruments: AHashSet::new(),
23600 }),
23601 )
23602 .unwrap();
23603
23604 assert_eq!(
23605 rig.dispatch
23606 .order_identity(&rig.cloid)
23607 .unwrap()
23608 .filled_qty(),
23609 Decimal::ZERO
23610 );
23611 assert_eq!(sweep.reports, vec![report.clone()]);
23612 assert!(sweep.covers_window);
23613 assert!(sweep.withheld_instruments.is_empty());
23614 assert!(!order_shape_pending(
23615 &client.core,
23616 &rig.dispatch,
23617 &rig.cloid
23618 ));
23619 assert_eq!(rig.dispatch.trade_source(fill.trade_id), None);
23620
23621 let repeated = finish_fill_reports(
23622 &client.core,
23623 &rig.dispatch,
23624 BTreeSet::new(),
23625 Ok(FillSweep {
23626 reports: vec![report.clone()],
23627 covers_window: true,
23628 withheld_instruments: AHashSet::new(),
23629 }),
23630 )
23631 .unwrap();
23632
23633 assert_eq!(repeated.reports, vec![report]);
23634 }
23635
23636 msgbus::publish_order_event(
23637 "events.order.S-001".into(),
23638 &OrderEventAny::Filled(fill.clone()),
23639 );
23640 assert_eq!(
23641 rig.dispatch
23642 .order_identity(&rig.cloid)
23643 .unwrap()
23644 .filled_qty(),
23645 Decimal::ZERO
23646 );
23647
23648 if !declined {
23649 cache
23650 .borrow_mut()
23651 .update_order(&OrderEventAny::Filled(fill.clone()))
23652 .unwrap();
23653 msgbus::publish_order_event(
23654 "events.order.S-001".into(),
23655 &OrderEventAny::Filled(fill.clone()),
23656 );
23657 order.apply(OrderEventAny::Filled(fill.clone())).unwrap();
23658 }
23659
23660 if restored {
23661 rig.dispatch = WsDispatchState::new();
23662 rig.dispatch.seed_order_restorations(vec![order.clone()]);
23663 }
23664
23665 if !restored {
23666 assert_eq!(
23667 rig.dispatch
23668 .order_identity(&rig.cloid)
23669 .unwrap()
23670 .filled_qty(),
23671 if declined { Decimal::ZERO } else { first.size }
23672 );
23673 }
23674
23675 frame.status = LighterOrderStatus::CanceledPostOnly;
23676 frame.filled_base_amount = Decimal::from_str("0.0030").unwrap();
23677 dispatch_lighter_order(
23678 &frame,
23679 &rig.dispatch,
23680 &rig.emitter,
23681 &rig.registry,
23682 account_id(),
23683 trader_id(),
23684 UnixNanos::from(4),
23685 );
23686 let report =
23687 parse_ws_order_status_report(&frame, &instrument, account_id(), UnixNanos::from(4))
23688 .unwrap()
23689 .with_client_order_id(rig.cloid);
23690 assert!(!order_report_ready(
23691 &client.core,
23692 &rig.dispatch,
23693 &report,
23694 &[]
23695 ));
23696 assert_eq!(
23697 execution_instruments_pending(&client.core, &rig.dispatch),
23698 AHashSet::from_iter([rig.instrument_id])
23699 );
23700 dispatch_lighter_trade(
23701 &first,
23702 &rig.dispatch,
23703 &rig.emitter,
23704 &rig.registry,
23705 account_id(),
23706 trader_id(),
23707 Some(TEST_ACCOUNT_INDEX_I64),
23708 UnixNanos::from(5),
23709 );
23710 let recovered = drain_events(&mut rig.rx);
23711
23712 if declined {
23713 assert_eq!(recovered.len(), 1);
23714
23715 let ExecutionEvent::Order(OrderEventAny::Filled(recovered_fill)) = &recovered[0] else {
23716 panic!("discarded HTTP report must remain recoverable as a real fill");
23717 };
23718
23719 assert_eq!(recovered_fill.trade_id, fill.trade_id);
23720 assert_eq!(recovered_fill.last_qty, fill.last_qty);
23721 cache
23722 .borrow_mut()
23723 .update_order(&OrderEventAny::Filled(recovered_fill.clone()))
23724 .unwrap();
23725 msgbus::publish_order_event(
23726 "events.order.S-001".into(),
23727 &OrderEventAny::Filled(recovered_fill.clone()),
23728 );
23729 order
23730 .apply(OrderEventAny::Filled(recovered_fill.clone()))
23731 .unwrap();
23732 } else {
23733 assert!(
23734 recovered.is_empty(),
23735 "historical fill must not repeat or release terminal"
23736 );
23737 }
23738
23739 let mut second = first;
23740 second.trade_id += 1;
23741 second.trade_id_str = Some(second.trade_id.to_string());
23742 second.size = Decimal::from_str("0.0020").unwrap();
23743 dispatch_lighter_trade(
23744 &second,
23745 &rig.dispatch,
23746 &rig.emitter,
23747 &rig.registry,
23748 account_id(),
23749 trader_id(),
23750 Some(TEST_ACCOUNT_INDEX_I64),
23751 UnixNanos::from(6),
23752 );
23753 let events = drain_events(&mut rig.rx);
23754 assert_eq!(events.len(), 2);
23755 assert!(matches!(
23756 &events[0],
23757 ExecutionEvent::Order(OrderEventAny::Filled(_))
23758 ));
23759 assert!(matches!(
23760 &events[1],
23761 ExecutionEvent::Order(OrderEventAny::Canceled(_))
23762 ));
23763
23764 for event in events {
23765 let ExecutionEvent::Order(event) = event else {
23766 panic!("expected order event");
23767 };
23768
23769 order.apply(event).unwrap();
23770 }
23771
23772 assert_eq!(order.status(), OrderStatus::Canceled);
23773 assert_eq!(order.filled_qty(), Quantity::from("0.0030"));
23774 assert_eq!(order.price(), Some(Price::from("2353.00")));
23775 assert_eq!(order.trade_ids().len(), 2);
23776 assert_eq!(
23777 rig.dispatch
23778 .order_identity(&rig.cloid)
23779 .unwrap()
23780 .filled_qty(),
23781 Decimal::from_str("0.0030").unwrap()
23782 );
23783 }
23784
23785 #[rstest]
23786 #[case::paged(false)]
23787 #[case::repeated_cursor(true)]
23788 #[tokio::test]
23789 async fn terminal_recovery_paginates_without_duplicate_fills(#[case] repeat_cursor: bool) {
23790 let mut rig = dispatcher_rig("TERMINAL-PAGES");
23791 register_identity(&rig);
23792 rig.dispatch.mark_accepted_emitted(rig.cloid);
23793 let mut order = dispatcher_test_order(&rig, LighterOrderStatus::Canceled);
23794 order.filled_base_amount = Decimal::from_str("0.0030").unwrap();
23795 let mut first = dispatcher_test_trade(&rig, true);
23796 first.bid_id = order.order_index;
23797 first.bid_id_str = Some(order.order_id.clone());
23798 first.size = Decimal::from_str("0.0010").unwrap();
23799 let mut second = first.clone();
23800 second.trade_id += 1;
23801 second.trade_id_str = Some(second.trade_id.to_string());
23802 second.size = Decimal::from_str("0.0020").unwrap();
23803 let hits = Arc::new(AtomicUsize::new(0));
23804 let count = hits.clone();
23805
23806 let app = Router::new().route("/api/v1/trades", get(move |axum::extract::Query(query): axum::extract::Query<BTreeMap<String, String>>| {
23807 let first = first.clone();
23808 let second = second.clone();
23809 let count = count.clone();
23810 async move {
23811 count.fetch_add(1, Ordering::Relaxed);
23812
23813 if let Some(cursor) = query.get("cursor") {
23814 assert_eq!(cursor, "older");
23815 axum::Json(serde_json::json!({"code":200,"trades":[first],"next_cursor":if repeat_cursor { Some("older") } else { None }}))
23816 } else {
23817 axum::Json(serde_json::json!({"code":200,"trades":[second],"next_cursor":"older"}))
23818 }
23819 }
23820 }));
23821
23822 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
23823 let addr = listener.local_addr().unwrap();
23824 let server = tokio::spawn(async move {
23825 axum::serve(listener, app).await.unwrap();
23826 });
23827
23828 let (client, _, _) = create_execution_client();
23829 let mut context = acked_probe_context(&client);
23830 context.http_client = LighterHttpClient::new(
23831 LighterEnvironment::Testnet,
23832 Some(format!("http://{addr}")),
23833 1,
23834 None,
23835 )
23836 .unwrap();
23837 context.dispatch = rig.dispatch.clone();
23838 context.registry = rig.registry.clone();
23839 context.emitter = rig.emitter.clone();
23840 dispatch_lighter_order(
23841 &order,
23842 &rig.dispatch,
23843 &rig.emitter,
23844 &rig.registry,
23845 account_id(),
23846 trader_id(),
23847 UnixNanos::from(1),
23848 );
23849 let result = recover_terminal_trades(order.market_index, order.order_index, &context).await;
23850 assert_eq!(hits.load(Ordering::Relaxed), 2);
23851
23852 if repeat_cursor {
23853 assert_eq!(
23854 result.unwrap_err().to_string(),
23855 "Lighter terminal recovery repeated cursor"
23856 );
23857 assert!(drain_events(&mut rig.rx).is_empty());
23858 assert!(
23859 rig.dispatch
23860 .terminals_pending
23861 .lock()
23862 .contains_key(&rig.cloid)
23863 );
23864 } else {
23865 result.unwrap();
23866 let events = drain_events(&mut rig.rx);
23867 assert_eq!(events.len(), 3);
23868 assert!(
23869 matches!(&events[0], ExecutionEvent::Order(OrderEventAny::Filled(fill)) if fill.last_qty == Quantity::from("0.0010"))
23870 );
23871 assert!(
23872 matches!(&events[1], ExecutionEvent::Order(OrderEventAny::Filled(fill)) if fill.last_qty == Quantity::from("0.0020"))
23873 );
23874 assert!(matches!(
23875 &events[2],
23876 ExecutionEvent::Order(OrderEventAny::Canceled(_))
23877 ));
23878 assert!(rig.dispatch.terminals_pending.lock().is_empty());
23879 }
23880
23881 server.abort();
23882 let _ = server.await;
23883 }
23884
23885 #[rstest]
23886 #[case::normal(false, 0)]
23887 #[case::interrupted_shutdown(true, 0)]
23888 #[case::interrupted_lookup(false, 1)]
23889 #[case::repeated_reconnect(false, ACKED_ORDER_PROBE_ATTEMPTS)]
23890 #[tokio::test]
23891 async fn terminal_shutdown_bounds_unresponsive_http(
23892 #[case] interrupted: bool,
23893 #[case] reconnects: usize,
23894 ) {
23895 let mut rig = dispatcher_rig("TERMINAL-TIMEOUT");
23896 register_identity(&rig);
23897 rig.dispatch.mark_accepted_emitted(rig.cloid);
23898 let mut order = dispatcher_test_order(&rig, LighterOrderStatus::CanceledExpired);
23899 order.filled_base_amount = Decimal::from_str("0.0025").unwrap();
23900 dispatch_lighter_order(
23901 &order,
23902 &rig.dispatch,
23903 &rig.emitter,
23904 &rig.registry,
23905 account_id(),
23906 trader_id(),
23907 UnixNanos::from(1),
23908 );
23909 let hits = Arc::new(AtomicUsize::new(0));
23910 let count = hits.clone();
23911 let app = Router::new().route(
23912 "/api/v1/trades",
23913 get(move || {
23914 count.fetch_add(1, Ordering::Relaxed);
23915 std::future::pending::<String>()
23916 }),
23917 );
23918
23919 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
23920 let addr = listener.local_addr().unwrap();
23921 let server = tokio::spawn(async move {
23922 axum::serve(listener, app).await.unwrap();
23923 });
23924
23925 let (mut client, _, _) = create_execution_client();
23926 client.http_client = LighterHttpClient::new(
23927 LighterEnvironment::Testnet,
23928 Some(format!("http://{addr}")),
23929 60,
23930 None,
23931 )
23932 .unwrap();
23933 client.dispatch = rig.dispatch.clone();
23934 client.registry = rig.registry.clone();
23935 client.emitter = rig.emitter.clone();
23936 let context = acked_probe_context(&client);
23937 client
23938 .terminal_recovery_handle
23939 .spawn(run_terminal_recovery(
23940 context,
23941 client.terminal_producers_stopped.clone(),
23942 ))
23943 .unwrap();
23944
23945 for expected in 1..=reconnects {
23946 rig.dispatch.terminals_recover.notify_one();
23947 wait_until_async(
23948 || async { hits.load(Ordering::Relaxed) == expected },
23949 Duration::from_secs(2),
23950 )
23951 .await;
23952 }
23953
23954 client.stop().unwrap();
23955
23956 if interrupted {
23957 {
23958 let shutdown = client.finish_session_shutdown();
23959 tokio::pin!(shutdown);
23960 tokio::select! {
23961 result = &mut shutdown => panic!("shutdown completed before blocked recovery: {result:?}"),
23962 () = wait_until_async(|| async { hits.load(Ordering::Relaxed) == 1 }, Duration::from_secs(2)) => {},
23963 }
23964 }
23965
23966 assert!(client.terminal_recovery_handle.is_some());
23967 assert!(client.terminal_producers_stopped.is_cancelled());
23968 }
23969
23970 tokio::time::timeout(
23971 TERMINAL_SHUTDOWN_TIMEOUT + DISCONNECT_TIMEOUT,
23972 client.finish_session_shutdown(),
23973 )
23974 .await
23975 .unwrap()
23976 .unwrap();
23977 let events = drain_events(&mut rig.rx);
23978 assert_eq!(hits.load(Ordering::Relaxed), ACKED_ORDER_PROBE_ATTEMPTS);
23979 assert_eq!(events.len(), 1);
23980 assert!(
23981 matches!(&events[0], ExecutionEvent::Order(OrderEventAny::Expired(event))
23982 if event.client_order_id == rig.cloid)
23983 );
23984 assert!(rig.dispatch.terminals_pending.lock().is_empty());
23985 assert!(client.terminal_recovery_handle.is_none());
23986 server.abort();
23987 let _ = server.await;
23988 }
23989
23990 #[rstest]
23991 #[case::query_terminal(0, 0)]
23992 #[case::query_deferred(0, 1)]
23993 #[case::cancel_terminal(1, 0)]
23994 #[case::cancel_deferred(1, 1)]
23995 #[case::query_partial(0, 2)]
23996 #[case::cancel_partial(1, 2)]
23997 #[case::create_partial(2, 2)]
23998 #[case::query_queued_fill(0, 3)]
23999 #[case::cancel_queued_fill(1, 3)]
24000 #[case::create_queued_fill(2, 3)]
24001 #[tokio::test]
24002 async fn terminal_blocks_http_order_publication(#[case] route: u8, #[case] phase: u8) {
24003 let (mut client, cache, mut rx) = create_execution_client();
24004 let instrument = register_test_instrument(&client, &cache);
24005 let id = ClientOrderId::from("TERMINAL-HTTP-BARRIER");
24006 let venue = VenueOrderId::from("123");
24007 let mut factory = test_order_factory();
24008 let cached = test_limit_order(&mut factory, instrument, id.as_str());
24009 let index = client.dispatch.register_create_identity(&cached).unwrap();
24010 client
24011 .dispatch
24012 .mark_order_submission(&id, TEST_SUBMISSION_NONCE);
24013 cache_accepted_order(&cache, cached, venue, Some(client_id()));
24014 client.dispatch.mark_accepted_emitted(id);
24015 let mut order = reconciliation_raw_order(index, venue, LighterOrderStatus::Canceled);
24016 order.filled_base_amount = Decimal::from_str("0.0025").unwrap();
24017 if phase >= 2 {
24018 order.status = LighterOrderStatus::Open;
24019 order.remaining_base_amount = order.initial_base_amount - order.filled_base_amount;
24020
24021 if phase == 3 {
24022 let mut report =
24023 reconciliation_fill_report(instrument, index, venue, "QUEUED-REAL-FILL");
24024 report.client_order_id = Some(id);
24025 report.last_qty = Quantity::from("0.0025");
24026 let identity = client.dispatch.order_identity(&id).unwrap();
24027 let cached = cache.borrow().order(&id).unwrap().clone();
24028 let inst = cache.borrow().instrument(&instrument).unwrap().clone();
24029
24030 let OrderEventAny::Filled(fill) = TestOrderEventStubs::filled(
24031 &cached,
24032 &inst,
24033 Some(report.trade_id),
24034 None,
24035 Some(report.last_px),
24036 Some(report.last_qty),
24037 None,
24038 Some(report.commission),
24039 None,
24040 Some(account_id()),
24041 ) else {
24042 panic!("expected fill");
24043 };
24044
24045 assert!(!identity.has_publication());
24046 dispatch_lighter_fill(
24047 fill,
24048 &client.emitter,
24049 &client.dispatch,
24050 &client.dispatch.order_events.lock(),
24051 );
24052 }
24053 } else {
24054 dispatch_lighter_order(
24055 &order,
24056 &client.dispatch,
24057 &client.emitter,
24058 &client.registry,
24059 account_id(),
24060 trader_id(),
24061 UnixNanos::from(1),
24062 );
24063 assert!(client.dispatch.terminals_pending.lock().contains_key(&id));
24064
24065 if phase == 1 {
24066 client.dispatch.terminals_pending.lock().remove(&id);
24067 client
24068 .dispatch
24069 .order_events
24070 .lock()
24071 .entry(id)
24072 .or_default()
24073 .store_order(&order);
24074 }
24075 }
24076
24077 let app = Router::new()
24078 .route(
24079 "/api/v1/accountActiveOrders",
24080 get(|| async { axum::Json(serde_json::json!({"code":200,"orders":[]})) }),
24081 )
24082 .route(
24083 "/api/v1/accountInactiveOrders",
24084 get(move || {
24085 let order = order.clone();
24086 async move { axum::Json(serde_json::json!({"code":200,"orders":[order]})) }
24087 }),
24088 );
24089
24090 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
24091 let addr = listener.local_addr().unwrap();
24092 let server = tokio::spawn(async move {
24093 axum::serve(listener, app).await.unwrap();
24094 });
24095
24096 client.http_client = LighterHttpClient::new(
24097 LighterEnvironment::Testnet,
24098 Some(format!("http://{addr}")),
24099 1,
24100 None,
24101 )
24102 .unwrap();
24103
24104 if route == 1 {
24105 enqueue_cancel(&client, instrument, id, venue, 12);
24106 let pending = client.dispatch.pop_pending_sendtx_head().unwrap();
24107 assert!(
24108 !recover_acked_action_report(&pending, &acked_probe_context(&client))
24109 .await
24110 .unwrap()
24111 );
24112 assert_eq!(client.dispatch.pending_cancel_nonce(&id), Some((0, 12)));
24113 } else if route == 2 {
24114 let cached = cache.borrow().order(&id).unwrap().clone();
24115
24116 let probe = AckedOrderProbe {
24117 order: Box::new(cached),
24118 client_order_index: index,
24119 connection_epoch: 0,
24120 nonce: TEST_SUBMISSION_NONCE,
24121 api_key_index: TEST_API_KEY_INDEX,
24122 tx_hash: "create-probe".to_string(),
24123 };
24124
24125 assert!(!probe_acked_create_report(&probe, &acked_probe_context(&client), 1).await);
24126 } else {
24127 client
24128 .query_order(QueryOrder::new(
24129 trader_id(),
24130 Some(client_id()),
24131 strategy_id(),
24132 instrument,
24133 id,
24134 Some(venue),
24135 UUID4::new(),
24136 UnixNanos::from(2),
24137 None,
24138 None,
24139 ))
24140 .unwrap();
24141 client.pending_tasks.begin_shutdown();
24142 client
24143 .pending_tasks
24144 .finish_shutdown(Duration::from_secs(2), DISCONNECT_TIMEOUT)
24145 .await
24146 .unwrap();
24147 }
24148
24149 let events = drain_events(&mut rx);
24150 assert_eq!(events.len(), usize::from(phase == 3));
24151 assert_eq!(
24152 cache.borrow().order(&id).unwrap().filled_qty(),
24153 Quantity::from("0.0000")
24154 );
24155
24156 if phase == 3 {
24157 assert!(matches!(
24158 &events[0],
24159 ExecutionEvent::Order(OrderEventAny::Filled(_))
24160 ));
24161 }
24162
24163 assert!(client.dispatch.order_identities.contains_key(&id));
24164 server.abort();
24165 let _ = server.await;
24166 }
24167
24168 #[rstest]
24169 #[case::normal(false, false)]
24170 #[case::interrupted(true, false)]
24171 #[case::incomplete(false, true)]
24172 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
24173 async fn terminal_shutdown_drains_after_producer_drop(
24174 #[case] interrupted: bool,
24175 #[case] incomplete: bool,
24176 ) {
24177 let mut rig = dispatcher_rig("TERMINAL-LATE-PRODUCER");
24178 register_identity(&rig);
24179 rig.dispatch.mark_accepted_emitted(rig.cloid);
24180 let mut order = dispatcher_test_order(&rig, LighterOrderStatus::CanceledExpired);
24181 let mut trade = dispatcher_test_trade(&rig, true);
24182 trade.bid_id = order.order_index;
24183 trade.bid_id_str = Some(order.order_id.clone());
24184 order.filled_base_amount = trade.size;
24185 let (http_client, hits, server) = terminal_trades_server(&order, vec![trade]).await;
24186 let (mut client, _, _) = create_execution_client();
24187 client.http_client = http_client;
24188 client.dispatch = rig.dispatch.clone();
24189 client.registry = rig.registry.clone();
24190 client.emitter = rig.emitter.clone();
24191 client.subscribe_applied_fills();
24192 let context = acked_probe_context(&client);
24193 client
24194 .terminal_recovery_handle
24195 .spawn(run_terminal_recovery(
24196 context.clone(),
24197 client.terminal_producers_stopped.clone(),
24198 ))
24199 .unwrap();
24200 let (ready, started) = tokio::sync::oneshot::channel();
24201 let (release, released) = std::sync::mpsc::sync_channel::<()>(1);
24202
24203 client
24204 .pending_tasks
24205 .spawn(async move {
24206 let _guard = TaskGroupGuard::new(&[], move || {
24207 dispatch_lighter_order(
24208 &order,
24209 &context.dispatch,
24210 &context.emitter,
24211 &context.registry,
24212 account_id(),
24213 trader_id(),
24214 UnixNanos::from(2),
24215 );
24216 });
24217
24218 ready.send(()).unwrap();
24219
24220 if incomplete {
24221 released.recv().unwrap();
24222 } else {
24223 std::future::pending::<()>().await;
24224 }
24225 })
24226 .unwrap();
24227
24228 started.await.unwrap();
24229 client.stop().unwrap();
24230
24231 if interrupted {
24232 assert!(
24233 tokio::time::timeout(Duration::from_millis(20), client.finish_session_shutdown())
24234 .await
24235 .is_err()
24236 );
24237 assert!(!client.terminal_producers_stopped.is_cancelled());
24238 assert!(client.terminal_recovery_handle.is_some());
24239 assert!(!client.pending_tasks.is_empty());
24240 }
24241
24242 if incomplete {
24243 assert!(client.finish_session_shutdown().await.is_err());
24244 assert!(!client.terminal_producers_stopped.is_cancelled());
24245 assert!(client.terminal_recovery_handle.is_some());
24246 assert!(!client.pending_tasks.is_empty());
24247 assert!(client.fills_applied.is_some());
24248 release.send(()).unwrap();
24249 }
24250
24251 client.finish_session_shutdown().await.unwrap();
24252 let events = drain_events(&mut rig.rx);
24253 assert_eq!(hits.load(Ordering::Relaxed), 1);
24254 assert_eq!(events.len(), 2);
24255 assert!(matches!(
24256 &events[0],
24257 ExecutionEvent::Order(OrderEventAny::Filled(_))
24258 ));
24259 assert!(matches!(
24260 &events[1],
24261 ExecutionEvent::Order(OrderEventAny::Expired(_))
24262 ));
24263 assert!(rig.dispatch.terminals_pending.lock().is_empty());
24264 assert!(client.session_tasks_finished());
24265 assert!(client.fills_applied.is_none());
24266 server.abort();
24267 let _ = server.await;
24268 }
24269
24270 #[rstest]
24271 #[case::canceled(LighterOrderStatus::Canceled, true, false, false)]
24272 #[case::expired(LighterOrderStatus::CanceledExpired, true, false, false)]
24273 #[case::missing_trade(LighterOrderStatus::Canceled, false, false, false)]
24274 #[case::canceled_bundle_first(LighterOrderStatus::Canceled, true, true, true)]
24275 #[case::canceled_mass_first(LighterOrderStatus::Canceled, true, true, false)]
24276 #[case::expired_bundle_first(LighterOrderStatus::CanceledExpired, true, true, true)]
24277 #[case::expired_mass_first(LighterOrderStatus::CanceledExpired, true, true, false)]
24278 #[case::early_missing_trade(LighterOrderStatus::Canceled, false, true, false)]
24279 #[tokio::test]
24280 async fn terminal_mass_status_pairs_missing_fills(
24281 #[case] status: LighterOrderStatus,
24282 #[case] complete: bool,
24283 #[case] early_timestamp: bool,
24284 #[case] bundle_first: bool,
24285 ) {
24286 let (mut rig, cache, mut engine, mut frame) = reduce_order_replay();
24287 cache
24288 .borrow_mut()
24289 .add_account(AccountAny::Margin(test_margin_account()))
24290 .unwrap();
24291 dispatch_reduce_frame(&rig, &frame, false);
24292 consume_reduce_events(&mut rig, &mut engine);
24293 let instrument = LIGHTER_INSTRUMENT_CACHE
24294 .get(&rig.instrument_id)
24295 .unwrap()
24296 .value()
24297 .clone();
24298 let mut first = dispatcher_test_trade(&rig, true);
24299 first.bid_id = frame.order_index;
24300 first.bid_id_str = Some(frame.order_id.clone());
24301 first.size = Decimal::from_str("0.0005").unwrap();
24302 first.usd_amount = first.size * first.price;
24303 let mut initial_fill = parse_ws_fill_report(
24304 &first,
24305 TEST_ACCOUNT_INDEX_I64,
24306 &instrument,
24307 account_id(),
24308 UnixNanos::from(2),
24309 )
24310 .unwrap()
24311 .unwrap();
24312 initial_fill.client_order_id = Some(rig.cloid);
24313 engine.reconcile_fill_report(&initial_fill);
24314 let mut second = first.clone();
24315 second.trade_id += 1;
24316 second.trade_id_str = Some(second.trade_id.to_string());
24317 second.timestamp += 1;
24318 second.size = Decimal::from_str("0.0010").unwrap();
24319 second.usd_amount = second.size * second.price;
24320 let second_id = parse_lighter_trade_id(&second).unwrap();
24321 frame.updated_at = if early_timestamp {
24322 second.timestamp - 2
24323 } else {
24324 second.timestamp + 1
24325 };
24326
24327 let terminal_timestamp = UnixNanos::from(frame.updated_at as u64 * 1_000_000);
24328 frame.status = status;
24329 frame.filled_base_amount =
24330 Decimal::from_str(if complete { "0.0015" } else { "0.0020" }).unwrap();
24331 frame.filled_quote_amount = frame.filled_base_amount * frame.price;
24332 frame.remaining_base_amount = Decimal::ZERO;
24333
24334 let mut client = create_execution_client_with_cache(cache.clone());
24335 client.registry = rig.registry.clone();
24336 client.emitter = rig.emitter.clone();
24337 client
24338 .dispatch
24339 .seed_order_restorations(vec![cache.borrow().order(&rig.cloid).unwrap().clone()]);
24340 client.dispatch.note_active_market(frame.market_index);
24341
24342 let position = PositionStatusReport::new(
24343 account_id(),
24344 rig.instrument_id,
24345 PositionSide::Long,
24346 Quantity::from("0.0015"),
24347 UnixNanos::from(3),
24348 UnixNanos::from(3),
24349 None,
24350 None,
24351 None,
24352 );
24353 client
24354 .dispatch
24355 .replace_position_snapshot(&[position], &[], &[]);
24356 let trades = vec![first, second.clone(), second];
24357
24358 let app = Router::new()
24359 .route(
24360 "/api/v1/accountActiveOrders",
24361 get(|| async { axum::Json(serde_json::json!({"code":200,"orders":[]})) }),
24362 )
24363 .route(
24364 "/api/v1/accountInactiveOrders",
24365 get(move || {
24366 let frame = frame.clone();
24367 async move { axum::Json(serde_json::json!({"code":200,"orders":[frame]})) }
24368 }),
24369 )
24370 .route(
24371 "/api/v1/trades",
24372 get(move || {
24373 let trades = trades.clone();
24374 async move { axum::Json(serde_json::json!({"code":200,"trades":trades})) }
24375 }),
24376 );
24377
24378 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
24379 let addr = listener.local_addr().unwrap();
24380 let server = tokio::spawn(async move {
24381 axum::serve(listener, app).await.unwrap();
24382 });
24383
24384 client.http_client = LighterHttpClient::new(
24385 LighterEnvironment::Testnet,
24386 Some(format!("http://{addr}")),
24387 1,
24388 None,
24389 )
24390 .unwrap();
24391 let mass = client.generate_mass_status(None).await.unwrap().unwrap();
24392 assert_eq!(
24393 mass.order_reports().len(),
24394 usize::from(complete && !early_timestamp)
24395 );
24396 let bundles = drain_events(&mut rig.rx);
24397 assert_eq!(bundles.len(), usize::from(complete && early_timestamp));
24398 assert_eq!(mass.fill_reports().values().map(Vec::len).sum::<usize>(), 2);
24399 assert_eq!(mass.position_reports().len(), usize::from(complete));
24400 let clock: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(VirtualClock::default()));
24401 let engine = Rc::new(RefCell::new(engine));
24402 engine
24403 .borrow_mut()
24404 .register_client(Box::new(client))
24405 .unwrap();
24406 let mut manager =
24407 ExecutionManager::new(clock, cache.clone(), ExecutionManagerConfig::default()).unwrap();
24408 let applied = Rc::new(RefCell::new(Vec::new()));
24409 let captured = applied.clone();
24410 let handler = TypedHandler::from(move |event: &OrderEventAny| {
24411 captured.borrow_mut().push(event.clone());
24412 });
24413
24414 msgbus::subscribe_order_events(ORDER_EVENTS_PATTERN.into(), handler.clone(), None);
24415 let subscription = AppliedFillSubscription(handler);
24416
24417 if bundle_first {
24418 for event in &bundles {
24419 let ExecutionEvent::Report(report) = event else {
24420 panic!("expected bundled report")
24421 };
24422
24423 engine.borrow_mut().reconcile_execution_report(report);
24424 }
24425 }
24426
24427 manager.reconcile_execution_mass_status(&mass, &engine);
24428
24429 if !bundle_first {
24430 for event in &bundles {
24431 let ExecutionEvent::Report(report) = event else {
24432 panic!("expected bundled report")
24433 };
24434
24435 engine.borrow_mut().reconcile_execution_report(report);
24436 }
24437 }
24438
24439 drop(subscription);
24440 let applied = applied.take();
24441 assert_eq!(applied.len(), if complete { 2 } else { 1 });
24442
24443 let OrderEventAny::Filled(fill) = &applied[0] else {
24444 panic!("expected real fill first")
24445 };
24446
24447 assert_eq!(fill.trade_id, second_id);
24448 assert_eq!(fill.last_qty, Quantity::from("0.0010"));
24449
24450 if complete {
24451 assert!(matches!(
24452 applied.last(),
24453 Some(OrderEventAny::Canceled(_) | OrderEventAny::Expired(_))
24454 ));
24455 assert_eq!(applied.last().unwrap().ts_event(), terminal_timestamp);
24456 }
24457
24458 let cached = cache.borrow().order(&rig.cloid).unwrap().clone();
24459 assert_eq!(cached.filled_qty(), Quantity::from("0.0015"));
24460 assert_eq!(cached.trade_ids(), vec![&initial_fill.trade_id, &second_id]);
24461 assert_eq!(
24462 cached.status(),
24463 if !complete {
24464 OrderStatus::PartiallyFilled
24465 } else if status == LighterOrderStatus::CanceledExpired {
24466 OrderStatus::Expired
24467 } else {
24468 OrderStatus::Canceled
24469 }
24470 );
24471 assert!(
24472 nautilus_execution::reconciliation::inferred_reconciliation_trade_ids(&cached)
24473 .unwrap()
24474 .is_empty()
24475 );
24476 server.abort();
24477 let _ = server.await;
24478 }
24479
24480 #[rstest]
24481 #[case::observed(0)]
24482 #[case::synced(1)]
24483 #[case::restored(2)]
24484 #[case::restored_child(3)]
24485 #[tokio::test]
24486 async fn terminal_inferred_fills_exhaust_without_replay(#[case] mode: u8) {
24487 let (mut rig, cache, mut engine, mut frame) = reduce_order_replay();
24488
24489 cache
24490 .borrow_mut()
24491 .add_account(AccountAny::Margin(test_margin_account()))
24492 .unwrap();
24493
24494 let mut client = create_execution_client_with_cache(cache.clone());
24495 client.dispatch = rig.dispatch.clone();
24496 client.emitter = rig.emitter.clone();
24497 client.registry = rig.registry.clone();
24498 if mode == 0 {
24499 client.subscribe_applied_fills();
24500 }
24501
24502 dispatch_reduce_frame(&rig, &frame, false);
24503 consume_reduce_events(&mut rig, &mut engine);
24504 frame.filled_base_amount = Decimal::from_str("0.0010").unwrap();
24505 frame.remaining_base_amount -= frame.filled_base_amount;
24506 let instrument = LIGHTER_INSTRUMENT_CACHE
24507 .get(&rig.instrument_id)
24508 .unwrap()
24509 .value()
24510 .clone();
24511 let mut report =
24512 parse_ws_order_status_report(&frame, &instrument, account_id(), UnixNanos::from(2))
24513 .unwrap()
24514 .with_client_order_id(rig.cloid);
24515 report.order_status = OrderStatus::PartiallyFilled;
24516 engine.reconcile_order_status_report(&report);
24517 let mut cached = cache.borrow().order(&rig.cloid).unwrap().clone();
24518 let inferred =
24519 nautilus_execution::reconciliation::inferred_reconciliation_trade_ids(&cached).unwrap();
24520 assert_eq!(inferred.len(), 1);
24521 assert_eq!(cached.filled_qty(), Quantity::from("0.0010"));
24522
24523 if mode == 1 {
24524 assert!(order_shape_pending(&client.core, &rig.dispatch, &rig.cloid));
24525 } else if mode >= 2 {
24526 if mode == 3 {
24527 let mut history = cached.events().into_iter().cloned().collect::<Vec<_>>();
24528
24529 let OrderEventAny::Initialized(init) = &mut history[0] else {
24530 panic!("expected initialization");
24531 };
24532
24533 init.parent_order_id = Some(ClientOrderId::new("INFERRED-PARENT"));
24534 cached = OrderAny::from_events(history).unwrap();
24535 cache
24536 .borrow_mut()
24537 .add_order(cached.clone(), None, Some(client_id()), true)
24538 .unwrap();
24539 }
24540
24541 rig.dispatch = WsDispatchState::new();
24542 rig.dispatch.seed_order_restorations(vec![cached]);
24543 client.dispatch = rig.dispatch.clone();
24544 dispatch_reduce_frame(&rig, &frame, false);
24545 assert!(drain_events(&mut rig.rx).is_empty());
24546 }
24547
24548 let positions_before = cache
24549 .borrow()
24550 .positions_open(None, None, None, None, None)
24551 .iter()
24552 .map(|position| position.quantity)
24553 .collect::<Vec<_>>();
24554 assert_eq!(positions_before, vec![Quantity::from("0.0010")]);
24555 let identity = rig.dispatch.order_identity(&rig.cloid).unwrap();
24556 assert!(identity.has_inferred_fills());
24557 assert_eq!(identity.filled_qty(), Decimal::ZERO);
24558 assert_eq!(rig.dispatch.trade_source(inferred[0]), None);
24559 assert_eq!(
24560 execution_instruments_pending(&client.core, &client.dispatch),
24561 AHashSet::from_iter([rig.instrument_id])
24562 );
24563 assert!(!order_report_ready(
24564 &client.core,
24565 &rig.dispatch,
24566 &report,
24567 &[]
24568 ));
24569 let mut trade = dispatcher_test_trade(&rig, true);
24570 trade.bid_id = frame.order_index;
24571 trade.bid_id_str = Some(frame.order_id.clone());
24572 trade.size = frame.filled_base_amount;
24573 let mut fill = parse_ws_fill_report(
24574 &trade,
24575 TEST_ACCOUNT_INDEX_I64,
24576 &instrument,
24577 account_id(),
24578 UnixNanos::from(3),
24579 )
24580 .unwrap()
24581 .unwrap();
24582 fill.client_order_id = Some(rig.cloid);
24583 frame.status = LighterOrderStatus::Canceled;
24584 frame.remaining_base_amount = Decimal::ZERO;
24585 dispatch_reduce_frame(&rig, &frame, false);
24586 assert!(drain_events(&mut rig.rx).is_empty());
24587 let (http, hits, server) = terminal_trades_server(&frame, vec![trade]).await;
24588 client.http_client = http;
24589 let context = acked_probe_context(&client);
24590
24591 if mode == 3 {
24592 let command = GenerateOrderStatusReport::new(
24593 UUID4::new(),
24594 UnixNanos::from(4),
24595 Some(rig.instrument_id),
24596 Some(rig.cloid),
24597 Some(VenueOrderId::new(&frame.order_id)),
24598 None,
24599 None,
24600 );
24601 assert!(client.generate_order_status_report(&command).await.is_err());
24602 wait_until_async(
24603 || async {
24604 client
24605 .dispatch
24606 .order_events
24607 .lock()
24608 .get(&rig.cloid)
24609 .is_none_or(|entry| entry.recovery_epoch.is_none())
24610 },
24611 Duration::from_secs(6),
24612 )
24613 .await;
24614 }
24615
24616 for attempt in 1..=ACKED_ORDER_PROBE_ATTEMPTS {
24617 recover_terminals(&context, true).await;
24618 let events = consume_reduce_events(&mut rig, &mut engine);
24619
24620 if attempt < ACKED_ORDER_PROBE_ATTEMPTS {
24621 assert!(events.is_empty());
24622 } else {
24623 assert_eq!(events.len(), 1);
24624 assert!(
24625 matches!(&events[0], OrderEventAny::Canceled(event) if event.ts_event == report.ts_last)
24626 );
24627 }
24628 }
24629
24630 let sweep = finish_fill_reports(
24631 &client.core,
24632 &rig.dispatch,
24633 BTreeSet::new(),
24634 Ok(FillSweep {
24635 reports: vec![fill],
24636 covers_window: true,
24637 withheld_instruments: AHashSet::new(),
24638 }),
24639 )
24640 .unwrap();
24641
24642 assert!(sweep.reports.is_empty());
24643 assert!(!sweep.covers_window);
24644 assert_eq!(
24645 sweep.withheld_instruments,
24646 AHashSet::from_iter([rig.instrument_id])
24647 );
24648 let cached = cache.borrow().order(&rig.cloid).unwrap().clone();
24649 assert_eq!(cached.filled_qty(), Quantity::from("0.0010"));
24650 assert_eq!(cached.trade_ids(), inferred.iter().collect::<Vec<_>>());
24651 assert_eq!(cached.status(), OrderStatus::Canceled);
24652 assert_eq!(
24653 cache
24654 .borrow()
24655 .positions_open(None, None, None, None, None)
24656 .iter()
24657 .map(|position| position.quantity)
24658 .collect::<Vec<_>>(),
24659 positions_before
24660 );
24661 assert_eq!(identity.filled_qty(), Decimal::ZERO);
24662 assert_eq!(hits.load(Ordering::Relaxed), ACKED_ORDER_PROBE_ATTEMPTS);
24663 assert!(rig.dispatch.terminals_pending.lock().is_empty());
24664 inferred_reconciliation_trade_ids(&cached).unwrap();
24665 assert_eq!(
24666 client
24667 .dispatch
24668 .order_events
24669 .lock()
24670 .get(&rig.cloid)
24671 .map_or(0, |entry| entry.fills.len()),
24672 0
24673 );
24674 client.dispatch.replace_position_snapshot(&[], &[], &[]);
24675 assert!(execution_instruments_pending(&client.core, &client.dispatch).is_empty());
24676
24677 let command = GeneratePositionStatusReports::new(
24678 UUID4::new(),
24679 UnixNanos::from(5),
24680 Some(rig.instrument_id),
24681 None,
24682 None,
24683 None,
24684 None,
24685 );
24686 assert!(
24687 client
24688 .generate_position_status_reports(&command)
24689 .await
24690 .unwrap()
24691 .is_empty()
24692 );
24693 let task = client
24694 .generate_position_status_reports_task(&command)
24695 .unwrap();
24696 task.collection.await;
24697 assert!((task.finish)().unwrap().is_empty());
24698 let clock: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(VirtualClock::default()));
24699
24700 let config = ExecutionManagerConfig {
24701 position_check_threshold_ns: DurationNanos::from_secs(0),
24702 ..ExecutionManagerConfig::default()
24703 };
24704
24705 let mut manager = ExecutionManager::new(clock, cache.clone(), config).unwrap();
24706 for event in manager.check_positions_consistency(&[&client]).await {
24707 engine.process(&event);
24708 }
24709
24710 assert_eq!(
24711 cache
24712 .borrow()
24713 .positions_open(None, None, None, None, None)
24714 .iter()
24715 .map(|position| position.signed_decimal_qty())
24716 .sum::<Decimal>(),
24717 Decimal::ZERO
24718 );
24719 assert_eq!(
24720 cache.borrow().order(&rig.cloid).unwrap().trade_ids(),
24721 inferred.iter().collect::<Vec<_>>()
24722 );
24723 assert_eq!(identity.filled_qty(), Decimal::ZERO);
24724 cache
24725 .borrow_mut()
24726 .purge_closed_orders(UnixNanos::from(u64::MAX), 0);
24727 assert!(cache.borrow().order(&rig.cloid).is_none());
24728 assert!(execution_instruments_pending(&client.core, &client.dispatch).is_empty());
24729 server.abort();
24730 let _ = server.await;
24731 }
24732
24733 #[rstest]
24734 #[case::applied(true, false)]
24735 #[case::discarded(false, false)]
24736 #[case::venue_uuid(true, true)]
24737 fn terminal_tracks_engine_applied_reconciliation(
24738 #[case] applied: bool,
24739 #[case] venue_uuid: bool,
24740 ) {
24741 let (mut rig, cache, mut engine, mut frame) = reduce_order_replay();
24742
24743 let mut client = create_execution_client_with_cache(cache.clone());
24744 client.dispatch = rig.dispatch.clone();
24745 client.emitter = rig.emitter.clone();
24746 client.subscribe_applied_fills();
24747 client.subscribe_applied_fills();
24748 dispatch_reduce_frame(&rig, &frame, false);
24749 consume_reduce_events(&mut rig, &mut engine);
24750 let mut trade = dispatcher_test_trade(&rig, true);
24751 trade.bid_id = frame.order_index;
24752 trade.bid_id_str = Some(frame.order_id.clone());
24753 trade.size = Decimal::from_str("0.0010").unwrap();
24754 if venue_uuid {
24755 trade.trade_id_str = Some("4b1da1f0-0d43-5a14-99b0-7245e76569af".to_string());
24756 }
24757
24758 let instrument = LIGHTER_INSTRUMENT_CACHE
24759 .get(&rig.instrument_id)
24760 .unwrap()
24761 .value()
24762 .clone();
24763 let mut report = parse_ws_fill_report(
24764 &trade,
24765 TEST_ACCOUNT_INDEX_I64,
24766 &instrument,
24767 account_id(),
24768 UnixNanos::from(2),
24769 )
24770 .unwrap()
24771 .unwrap();
24772 report.client_order_id = Some(rig.cloid);
24773
24774 for _ in 0..2 {
24775 let reports = finish_fill_reports(
24776 &client.core,
24777 &rig.dispatch,
24778 BTreeSet::new(),
24779 Ok(FillSweep {
24780 reports: vec![report.clone()],
24781 covers_window: true,
24782 withheld_instruments: AHashSet::new(),
24783 }),
24784 )
24785 .unwrap();
24786
24787 assert_eq!(reports.reports, vec![report.clone()]);
24788 assert!(!order_shape_pending(
24789 &client.core,
24790 &rig.dispatch,
24791 &rig.cloid
24792 ));
24793 }
24794
24795 if applied {
24796 engine.reconcile_fill_report(&report);
24797 }
24798
24799 let identity = rig.dispatch.order_identity(&rig.cloid).unwrap();
24800 assert_eq!(
24801 identity.filled_qty(),
24802 if applied { trade.size } else { Decimal::ZERO }
24803 );
24804 assert_eq!(
24805 rig.dispatch.trade_source(report.trade_id),
24806 if applied {
24807 Some(TradeDedupSource::Reconciliation)
24808 } else {
24809 None
24810 }
24811 );
24812 assert!(!identity.has_publication());
24813 frame.status = LighterOrderStatus::Canceled;
24814 frame.filled_base_amount = trade.size;
24815 frame.remaining_base_amount = Decimal::ZERO;
24816 dispatch_reduce_frame(&rig, &frame, false);
24817
24818 if !applied {
24819 assert!(drain_events(&mut rig.rx).is_empty());
24820 dispatch_lighter_trade(
24821 &trade,
24822 &rig.dispatch,
24823 &rig.emitter,
24824 &rig.registry,
24825 account_id(),
24826 trader_id(),
24827 Some(TEST_ACCOUNT_INDEX_I64),
24828 UnixNanos::from(3),
24829 );
24830 }
24831
24832 let events = consume_reduce_events(&mut rig, &mut engine);
24833 assert_eq!(events.len(), if applied { 1 } else { 2 });
24834 assert!(matches!(events.last(), Some(OrderEventAny::Canceled(_))));
24835
24836 if !applied {
24837 assert!(
24838 matches!(&events[0], OrderEventAny::Filled(fill) if fill.trade_id == report.trade_id)
24839 );
24840 }
24841
24842 let order = cache.borrow().order(&rig.cloid).unwrap().clone();
24843 assert_eq!(order.status(), OrderStatus::Canceled);
24844 assert_eq!(order.filled_qty(), Quantity::from("0.0010"));
24845 assert_eq!(order.trade_ids(), vec![&report.trade_id]);
24846 assert!(rig.dispatch.terminals_pending.lock().is_empty());
24847 frame.filled_base_amount += Decimal::from_str("0.0001").unwrap();
24848 dispatch_reduce_frame(&rig, &frame, false);
24849 assert!(drain_events(&mut rig.rx).is_empty());
24850 }
24851
24852 #[rstest]
24853 #[case::drop(true)]
24854 #[case::shutdown(false)]
24855 #[tokio::test]
24856 async fn terminal_fill_subscription_detaches(#[case] drop_client: bool) {
24857 let (mut rig, cache, mut engine, frame) = reduce_order_replay();
24858
24859 let mut client = create_execution_client_with_cache(cache.clone());
24860 client.dispatch = rig.dispatch.clone();
24861 client.subscribe_applied_fills();
24862 client.subscribe_applied_fills();
24863 dispatch_reduce_frame(&rig, &frame, false);
24864 consume_reduce_events(&mut rig, &mut engine);
24865 let mut report = reconciliation_fill_report(
24866 rig.instrument_id,
24867 frame.client_order_index,
24868 VenueOrderId::new(&frame.order_id),
24869 "APPLIED-FIRST",
24870 );
24871 report.client_order_id = Some(rig.cloid);
24872 report.last_qty = Quantity::from("0.0010");
24873 engine.reconcile_fill_report(&report);
24874 let identity = rig.dispatch.order_identity(&rig.cloid).unwrap();
24875 assert_eq!(identity.filled_qty(), Decimal::from_str("0.0010").unwrap());
24876
24877 if !drop_client {
24878 client.stop().unwrap();
24879 client.finish_session_shutdown().await.unwrap();
24880 assert!(client.fills_applied.is_none());
24881 }
24882
24883 drop(client);
24884 report.trade_id = TradeId::new("APPLIED-SECOND");
24885 engine.reconcile_fill_report(&report);
24886 assert_eq!(
24887 cache.borrow().order(&rig.cloid).unwrap().filled_qty(),
24888 Quantity::from("0.0020")
24889 );
24890 assert_eq!(identity.filled_qty(), Decimal::from_str("0.0010").unwrap());
24891 assert_eq!(rig.dispatch.trade_source(report.trade_id), None);
24892 }
24893
24894 fn test_margin_account() -> MarginAccount {
24895 MarginAccount::new(
24896 AccountState::new(
24897 account_id(),
24898 AccountType::Margin,
24899 vec![AccountBalance::new(
24900 Money::from("10000.00000000 USDC"),
24901 Money::from("0.00000000 USDC"),
24902 Money::from("10000.00000000 USDC"),
24903 )],
24904 Vec::new(),
24905 true,
24906 UUID4::new(),
24907 UnixNanos::from(1),
24908 UnixNanos::from(1),
24909 Some(Currency::from("USDC")),
24910 ),
24911 true,
24912 )
24913 }
24914
24915 #[rstest]
24916 #[case::inline_cancel(0, LighterOrderStatus::Canceled)]
24917 #[case::worker_cancel(1, LighterOrderStatus::Canceled)]
24918 #[case::query_cancel(2, LighterOrderStatus::Canceled)]
24919 #[case::inline_expire(0, LighterOrderStatus::CanceledExpired)]
24920 #[case::worker_expire(1, LighterOrderStatus::CanceledExpired)]
24921 #[case::query_expire(2, LighterOrderStatus::CanceledExpired)]
24922 #[case::query_filled(2, LighterOrderStatus::Filled)]
24923 #[tokio::test]
24924 async fn terminal_query_recovers_missed_frames(
24925 #[case] route: u8,
24926 #[case] status: LighterOrderStatus,
24927 ) {
24928 let (mut rig, cache, mut engine, mut frame) = reduce_order_replay();
24929 cache
24930 .borrow_mut()
24931 .add_account(AccountAny::Margin(test_margin_account()))
24932 .unwrap();
24933 dispatch_reduce_frame(&rig, &frame, false);
24934 consume_reduce_events(&mut rig, &mut engine);
24935
24936 let mut client = create_execution_client_with_cache(cache.clone());
24937 client.dispatch = rig.dispatch.clone();
24938 client.registry = rig.registry.clone();
24939 client.emitter = rig.emitter.clone();
24940 frame.status = status;
24941 let filled = status == LighterOrderStatus::Filled;
24942 let qty = Quantity::from(if filled { "0.0027" } else { "0.0010" });
24943 frame.filled_base_amount = qty.as_decimal();
24944 frame.filled_quote_amount = frame.filled_base_amount * frame.price;
24945 frame.remaining_base_amount = Decimal::ZERO;
24946 let mut trade = dispatcher_test_trade(&rig, true);
24947 trade.bid_id = frame.order_index;
24948 trade.bid_id_str = Some(frame.order_id.clone());
24949 trade.size = frame.filled_base_amount;
24950 trade.usd_amount = trade.size * trade.price;
24951 let trade_id = parse_lighter_trade_id(&trade).unwrap();
24952 let terminal_timestamp = UnixNanos::from(frame.updated_at as u64 * 1_000_000);
24953 let venue = VenueOrderId::from(frame.order_id.as_str());
24954 let raw = frame.clone();
24955 let hits = Arc::new(AtomicUsize::new(0));
24956 let counted = hits.clone();
24957
24958 let app = Router::new()
24959 .route(
24960 "/api/v1/accountActiveOrders",
24961 get(|| async { axum::Json(serde_json::json!({"code":200,"orders":[]})) }),
24962 )
24963 .route(
24964 "/api/v1/accountInactiveOrders",
24965 get(move || {
24966 let raw = raw.clone();
24967 async move { axum::Json(serde_json::json!({"code":200,"orders":[raw]})) }
24968 }),
24969 )
24970 .route(
24971 "/api/v1/trades",
24972 get(move || {
24973 let trade = trade.clone();
24974 let counted = counted.clone();
24975 async move {
24976 counted.fetch_add(1, Ordering::Relaxed);
24977 axum::Json(serde_json::json!({"code":200,"trades":[trade]}))
24978 }
24979 }),
24980 );
24981
24982 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
24983 let addr = listener.local_addr().unwrap();
24984 let server = tokio::spawn(async move {
24985 axum::serve(listener, app).await.unwrap();
24986 });
24987
24988 client.http_client = LighterHttpClient::new(
24989 LighterEnvironment::Testnet,
24990 Some(format!("http://{addr}")),
24991 1,
24992 None,
24993 )
24994 .unwrap();
24995
24996 let command = GenerateOrderStatusReport::new(
24997 UUID4::new(),
24998 UnixNanos::from(4),
24999 Some(rig.instrument_id),
25000 Some(rig.cloid),
25001 Some(venue),
25002 None,
25003 None,
25004 );
25005
25006 if filled && route == 2 {
25007 dispatch_reduce_frame(&rig, &frame, false);
25008 assert!(!client.dispatch.order_identities.contains_key(&rig.cloid));
25009 }
25010
25011 let context = acked_probe_context(&client);
25012 let report = match route {
25013 0 => client.generate_order_status_report(&command).await.unwrap(),
25014 1 => {
25015 let task = client.generate_order_status_report_task(&command).unwrap();
25016 task.collection.await;
25017 (task.finish)().unwrap()
25018 }
25019 _ => {
25020 client
25021 .query_order(QueryOrder::new(
25022 trader_id(),
25023 Some(client_id()),
25024 strategy_id(),
25025 rig.instrument_id,
25026 rig.cloid,
25027 Some(venue),
25028 UUID4::new(),
25029 UnixNanos::from(4),
25030 None,
25031 None,
25032 ))
25033 .unwrap();
25034 client.pending_tasks.begin_shutdown();
25035 client
25036 .pending_tasks
25037 .finish_shutdown(Duration::from_secs(2), DISCONNECT_TIMEOUT)
25038 .await
25039 .unwrap();
25040 None
25041 }
25042 };
25043
25044 if !filled || route != 2 {
25045 assert!(drain_events(&mut rig.rx).is_empty());
25046 }
25047
25048 assert_eq!(
25049 client
25050 .dispatch
25051 .terminals_pending
25052 .lock()
25053 .contains_key(&rig.cloid),
25054 route == 2 && !filled
25055 );
25056 let applied = Rc::new(RefCell::new(Vec::new()));
25057 let captured = applied.clone();
25058 let handler = TypedHandler::from(move |event: &OrderEventAny| {
25059 captured.borrow_mut().push(event.clone());
25060 });
25061
25062 msgbus::subscribe_order_events(ORDER_EVENTS_PATTERN.into(), handler.clone(), None);
25063 let subscription = AppliedFillSubscription(handler);
25064
25065 if let Some(report) = report {
25066 assert_eq!(report.filled_qty, qty);
25067
25068 let command = GenerateFillReports::new(
25069 UUID4::new(),
25070 UnixNanos::from(5),
25071 Some(rig.instrument_id),
25072 Some(venue),
25073 None,
25074 None,
25075 None,
25076 None,
25077 );
25078 let fills = client.generate_fill_reports(command).await.unwrap();
25079 let bundle = EngineExecutionReport::OrderWithFills(Box::new(report), fills);
25080 engine.reconcile_execution_report(&bundle);
25081 } else {
25082 if !filled {
25083 recover_terminals(&context, true).await;
25084 }
25085
25086 consume_reduce_events(&mut rig, &mut engine);
25087 }
25088
25089 drop(subscription);
25090 let events = applied.take();
25091 assert_eq!(events.len(), if filled { 1 } else { 2 });
25092
25093 let OrderEventAny::Filled(fill) = &events[0] else {
25094 panic!("expected real fill first");
25095 };
25096
25097 assert_eq!(fill.trade_id, trade_id);
25098 assert_eq!(fill.last_qty, qty);
25099
25100 if !filled {
25101 assert!(matches!(
25102 events[1],
25103 OrderEventAny::Canceled(_) | OrderEventAny::Expired(_)
25104 ));
25105 assert_eq!(events[1].ts_event(), terminal_timestamp);
25106 }
25107
25108 let cached = cache.borrow().order(&rig.cloid).unwrap().clone();
25109 assert_eq!(cached.trade_ids(), vec![&trade_id]);
25110 assert_eq!(cached.filled_qty(), qty);
25111 assert_eq!(hits.load(Ordering::Relaxed), 1);
25112 server.abort();
25113 let _ = server.await;
25114 }
25115
25116 #[rstest]
25117 #[case::bulk_complete(true, 2)]
25118 #[case::targeted_complete(false, 2)]
25119 #[case::bulk_partial(true, 1)]
25120 #[case::targeted_partial(false, 1)]
25121 #[case::bulk_empty(true, 0)]
25122 #[case::targeted_empty(false, 0)]
25123 #[tokio::test]
25124 async fn terminal_targeted_reconciliation_recovers_offsetting_fills(
25125 #[case] bulk_report: bool,
25126 #[case] coverage: u8,
25127 ) {
25128 let (mut rig, cache, mut engine, mut buy) = reduce_order_replay();
25129 cache
25130 .borrow_mut()
25131 .add_account(AccountAny::Margin(test_margin_account()))
25132 .unwrap();
25133 dispatch_reduce_frame(&rig, &buy, false);
25134 consume_reduce_events(&mut rig, &mut engine);
25135 let sell_id = ClientOrderId::from("MISSED-SELL");
25136 let sell_order = OrderTestBuilder::new(OrderType::Limit)
25137 .trader_id(trader_id())
25138 .strategy_id(strategy_id())
25139 .instrument_id(rig.instrument_id)
25140 .client_order_id(sell_id)
25141 .side(OrderSide::Sell)
25142 .quantity(Quantity::from("0.0027"))
25143 .price(Price::from("2352.74"))
25144 .build();
25145 let sell_index = rig.dispatch.register_create_identity(&sell_order).unwrap();
25146 rig.dispatch
25147 .mark_order_submission(&sell_id, TEST_SUBMISSION_NONCE + 1);
25148 cache
25149 .borrow_mut()
25150 .add_order(sell_order, None, Some(client_id()), false)
25151 .unwrap();
25152 engine.process(&OrderEventAny::Submitted(OrderSubmitted::new(
25153 trader_id(),
25154 strategy_id(),
25155 rig.instrument_id,
25156 sell_id,
25157 account_id(),
25158 UUID4::new(),
25159 UnixNanos::from(1),
25160 UnixNanos::from(1),
25161 )));
25162 let mut sell = buy.clone();
25163 sell.order_index += 1;
25164 sell.order_id = sell.order_index.to_string();
25165 sell.client_order_index = sell_index;
25166 sell.client_order_id = sell_index.to_string();
25167 sell.is_ask = true;
25168 sell.side = Some(LighterOrderSide::Sell);
25169 dispatch_reduce_frame(&rig, &sell, false);
25170 consume_reduce_events(&mut rig, &mut engine);
25171 buy.status = LighterOrderStatus::Canceled;
25172 sell.status = LighterOrderStatus::CanceledExpired;
25173 for frame in [&mut buy, &mut sell] {
25174 frame.filled_base_amount = Decimal::from_str("0.0010").unwrap();
25175 frame.filled_quote_amount = frame.filled_base_amount * frame.price;
25176 frame.remaining_base_amount = Decimal::ZERO;
25177 }
25178
25179 let mut buy_trade = dispatcher_test_trade(&rig, true);
25180 buy_trade.bid_id = buy.order_index;
25181 buy_trade.bid_id_str = Some(buy.order_id.clone());
25182
25183 let qty = Quantity::from(match coverage {
25184 0 => "0.0000",
25185 1 => "0.0005",
25186 _ => "0.0010",
25187 });
25188
25189 buy_trade.size = qty.as_decimal();
25190 buy_trade.usd_amount = buy_trade.size * buy_trade.price;
25191 let mut sell_trade = dispatcher_test_trade(&rig, false);
25192 sell_trade.trade_id += 1;
25193 sell_trade.trade_id_str = Some(sell_trade.trade_id.to_string());
25194 sell_trade.ask_id = sell.order_index;
25195 sell_trade.ask_id_str = Some(sell.order_id.clone());
25196 sell_trade.ask_client_id = sell_index;
25197 sell_trade.ask_client_id_str = Some(sell_index.to_string());
25198 sell_trade.size = buy_trade.size;
25199 sell_trade.usd_amount = sell_trade.size * sell_trade.price;
25200 let expected = [
25201 (
25202 rig.cloid,
25203 parse_lighter_trade_id(&buy_trade).unwrap(),
25204 OrderStatus::Canceled,
25205 ),
25206 (
25207 sell_id,
25208 parse_lighter_trade_id(&sell_trade).unwrap(),
25209 OrderStatus::Expired,
25210 ),
25211 ];
25212 let orders = vec![buy, sell];
25213
25214 let trades = if coverage == 0 {
25215 Vec::new()
25216 } else {
25217 vec![buy_trade, sell_trade]
25218 };
25219
25220 let history_hits = Arc::new(AtomicUsize::new(0));
25221 let hits = Arc::new(AtomicUsize::new(0));
25222 let counted = hits.clone();
25223
25224 let app = Router::new()
25225 .route(
25226 "/api/v1/accountActiveOrders",
25227 get(|| async { axum::Json(serde_json::json!({"code":200,"orders":[]})) }),
25228 )
25229 .route(
25230 "/api/v1/accountInactiveOrders",
25231 get(move || {
25232 let orders = orders.clone();
25233 let history_hits = history_hits.clone();
25234 async move {
25235 let first = history_hits.fetch_add(1, Ordering::Relaxed) == 0;
25236
25237 let orders = if !bulk_report && first {
25238 Vec::new()
25239 } else {
25240 orders
25241 };
25242
25243 axum::Json(serde_json::json!({"code":200,"orders":orders}))
25244 }
25245 }),
25246 )
25247 .route(
25248 "/api/v1/trades",
25249 get(move || {
25250 let trades = trades.clone();
25251 let counted = counted.clone();
25252 async move {
25253 counted.fetch_add(1, Ordering::Relaxed);
25254 axum::Json(serde_json::json!({"code":200,"trades":trades}))
25255 }
25256 }),
25257 );
25258
25259 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
25260 let addr = listener.local_addr().unwrap();
25261 let server = tokio::spawn(async move {
25262 axum::serve(listener, app).await.unwrap();
25263 });
25264
25265 let mut client = create_execution_client_with_cache(cache.clone());
25266 client.dispatch = rig.dispatch.clone();
25267 client.registry = rig.registry.clone();
25268 client.emitter = rig.emitter.clone();
25269 client.dispatch.note_active_market(TEST_MARKET_INDEX);
25270 client.dispatch.replace_position_snapshot(&[], &[], &[]);
25271 client.http_client = LighterHttpClient::new(
25272 LighterEnvironment::Testnet,
25273 Some(format!("http://{addr}")),
25274 1,
25275 None,
25276 )
25277 .unwrap();
25278 let clock: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(VirtualClock::default()));
25279
25280 let config = ExecutionManagerConfig {
25281 open_check_lookback_mins: None,
25282 open_check_missing_retries: 0,
25283 open_check_open_only: false,
25284 open_check_threshold_ns: DurationNanos::from_secs(0),
25285 single_order_query_delay_ms: 0,
25286 ..ExecutionManagerConfig::default()
25287 };
25288
25289 let mut manager = ExecutionManager::new(clock, cache.clone(), config).unwrap();
25290 assert!(
25291 cache
25292 .borrow()
25293 .positions_open(None, None, None, None, None)
25294 .is_empty()
25295 );
25296 let events = manager.check_open_orders(&[&client]).await;
25297 for event in &events {
25298 engine.process(event);
25299 }
25300
25301 assert_eq!(hits.load(Ordering::Relaxed), 2);
25302 assert_eq!(events.len(), usize::from(coverage) * 2);
25303
25304 for (id, trade_id, status) in expected {
25305 let order_events = events
25306 .iter()
25307 .filter(|event| event.client_order_id() == id)
25308 .collect::<Vec<_>>();
25309
25310 if coverage > 0 {
25311 assert!(
25312 matches!(order_events[0], OrderEventAny::Filled(fill) if fill.trade_id == trade_id && fill.last_qty == qty)
25313 );
25314 }
25315
25316 if coverage == 2 {
25317 assert!(matches!(
25318 order_events[1],
25319 OrderEventAny::Canceled(_) | OrderEventAny::Expired(_)
25320 ));
25321 }
25322
25323 let order = cache.borrow().order(&id).unwrap().clone();
25324 assert_eq!(
25325 order.status(),
25326 match coverage {
25327 0 => OrderStatus::Accepted,
25328 1 => OrderStatus::PartiallyFilled,
25329 _ => status,
25330 }
25331 );
25332 assert_eq!(order.filled_qty(), qty);
25333 let trade_ids: Vec<_> = std::iter::once(&trade_id)
25334 .take(usize::from(coverage > 0))
25335 .collect();
25336 assert_eq!(order.trade_ids(), trade_ids);
25337 assert!(
25338 inferred_reconciliation_trade_ids(&order)
25339 .unwrap()
25340 .is_empty()
25341 );
25342 }
25343
25344 assert_eq!(
25345 cache
25346 .borrow()
25347 .positions_open(None, None, None, None, None)
25348 .iter()
25349 .map(|position| position.signed_decimal_qty())
25350 .sum::<Decimal>(),
25351 Decimal::ZERO
25352 );
25353 assert!(drain_events(&mut rig.rx).is_empty());
25354 server.abort();
25355 let _ = server.await;
25356 }
25357
25358 #[rstest]
25359 #[case::bulk_partial(true, false)]
25360 #[case::targeted_partial(false, false)]
25361 #[case::bulk_filled(true, true)]
25362 #[case::targeted_filled(false, true)]
25363 #[tokio::test]
25364 async fn terminal_reconciliation_preserves_real_fill_gate(
25365 #[case] bulk: bool,
25366 #[case] filled: bool,
25367 ) {
25368 let (mut rig, cache, mut engine, mut frame) = reduce_order_replay();
25369 cache
25370 .borrow_mut()
25371 .add_account(AccountAny::Margin(test_margin_account()))
25372 .unwrap();
25373 dispatch_reduce_frame(&rig, &frame, false);
25374 consume_reduce_events(&mut rig, &mut engine);
25375
25376 frame.status = if filled {
25377 LighterOrderStatus::Filled
25378 } else {
25379 LighterOrderStatus::Open
25380 };
25381
25382 let qty = Quantity::from(if filled { "0.0027" } else { "0.0010" });
25383 frame.filled_base_amount = qty.as_decimal();
25384 frame.filled_quote_amount = frame.filled_base_amount * frame.price;
25385 frame.remaining_base_amount = frame.initial_base_amount - frame.filled_base_amount;
25386 let mut trade = dispatcher_test_trade(&rig, true);
25387 trade.bid_id = frame.order_index;
25388 trade.bid_id_str = Some(frame.order_id.clone());
25389 trade.size = frame.filled_base_amount;
25390 trade.usd_amount = trade.size * trade.price;
25391 let trade_id = parse_lighter_trade_id(&trade).unwrap();
25392 let response_trade = trade.clone();
25393 let trade_hits = Arc::new(AtomicUsize::new(0));
25394 let hits = trade_hits.clone();
25395 let active = frame.clone();
25396 let inactive = frame.clone();
25397 let history_hits = Arc::new(AtomicUsize::new(0));
25398 let active_hits = history_hits.clone();
25399
25400 let app = Router::new()
25401 .route(
25402 "/api/v1/trades",
25403 get(move || {
25404 let trade = response_trade.clone();
25405 let hits = hits.clone();
25406 async move {
25407 hits.fetch_add(1, Ordering::Relaxed);
25408 axum::Json(serde_json::json!({"code":200,"trades":[trade]}))
25409 }
25410 }),
25411 )
25412 .route(
25413 "/api/v1/accountActiveOrders",
25414 get(move || {
25415 let active = active.clone();
25416 let hits = active_hits.clone();
25417 async move {
25418 let orders =
25419 if !filled && (bulk || hits.fetch_add(1, Ordering::Relaxed) > 0) {
25420 vec![active]
25421 } else {
25422 Vec::new()
25423 };
25424
25425 axum::Json(serde_json::json!({"code":200,"orders":orders}))
25426 }
25427 }),
25428 )
25429 .route(
25430 "/api/v1/accountInactiveOrders",
25431 get(move || {
25432 let inactive = inactive.clone();
25433 let hits = history_hits.clone();
25434 async move {
25435 let orders = if filled && (bulk || hits.fetch_add(1, Ordering::Relaxed) > 0)
25436 {
25437 vec![inactive]
25438 } else {
25439 Vec::new()
25440 };
25441
25442 axum::Json(serde_json::json!({"code":200,"orders":orders}))
25443 }
25444 }),
25445 );
25446
25447 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
25448 let addr = listener.local_addr().unwrap();
25449 let server = tokio::spawn(async move {
25450 axum::serve(listener, app).await.unwrap();
25451 });
25452
25453 let mut client = create_execution_client_with_cache(cache.clone());
25454 client.dispatch = rig.dispatch.clone();
25455 client.registry = rig.registry.clone();
25456 client.emitter = rig.emitter.clone();
25457 client.dispatch.note_active_market(TEST_MARKET_INDEX);
25458 client.http_client = LighterHttpClient::new(
25459 LighterEnvironment::Testnet,
25460 Some(format!("http://{addr}")),
25461 1,
25462 None,
25463 )
25464 .unwrap();
25465 client.subscribe_applied_fills();
25466 let clock: Rc<RefCell<dyn Clock>> = Rc::new(RefCell::new(VirtualClock::default()));
25467
25468 let config = ExecutionManagerConfig {
25469 open_check_lookback_mins: None,
25470 open_check_missing_retries: 0,
25471 open_check_open_only: false,
25472 open_check_threshold_ns: DurationNanos::from_secs(0),
25473 single_order_query_delay_ms: 0,
25474 ..ExecutionManagerConfig::default()
25475 };
25476
25477 let mut manager = ExecutionManager::new(clock, cache.clone(), config).unwrap();
25478 let events = manager.check_open_orders(&[&client]).await;
25479 assert!(
25480 events.is_empty(),
25481 "uncovered reports must not create inferred fills"
25482 );
25483
25484 let command = GenerateOrderStatusReport::new(
25485 UUID4::new(),
25486 UnixNanos::from(4),
25487 Some(rig.instrument_id),
25488 Some(rig.cloid),
25489 Some(VenueOrderId::new(&frame.order_id)),
25490 None,
25491 None,
25492 );
25493 assert!(client.generate_order_status_report(&command).await.is_err());
25494 let task = client.generate_order_status_report_task(&command).unwrap();
25495 task.collection.await;
25496 assert!((task.finish)().is_err());
25497 assert_eq!(rig.rx.len(), usize::from(filled));
25498 assert_eq!(trade_hits.load(Ordering::Relaxed), usize::from(filled));
25499 dispatch_lighter_trade(
25500 &trade,
25501 &client.dispatch,
25502 &client.emitter,
25503 &client.registry,
25504 account_id(),
25505 trader_id(),
25506 Some(TEST_ACCOUNT_INDEX_I64),
25507 UnixNanos::from(5),
25508 );
25509 consume_reduce_events(&mut rig, &mut engine);
25510 let report = client
25511 .generate_order_status_report(&command)
25512 .await
25513 .unwrap()
25514 .unwrap();
25515 assert_eq!(report.filled_qty, qty);
25516 let cached = cache.borrow().order(&rig.cloid).unwrap().clone();
25517 assert_eq!(cached.trade_ids(), vec![&trade_id]);
25518 assert_eq!(cached.filled_qty(), qty);
25519 assert_eq!(
25520 cached.status(),
25521 if filled {
25522 OrderStatus::Filled
25523 } else {
25524 OrderStatus::PartiallyFilled
25525 }
25526 );
25527 assert!(
25528 inferred_reconciliation_trade_ids(&cached)
25529 .unwrap()
25530 .is_empty()
25531 );
25532 server.abort();
25533 let _ = server.await;
25534 }
25535
25536 #[rstest]
25537 #[case::canceled(LighterOrderStatus::Canceled)]
25538 #[case::filled(LighterOrderStatus::Filled)]
25539 #[tokio::test]
25540 async fn terminal_mass_status_recovers_child_without_sweep_fills(
25541 #[case] status: LighterOrderStatus,
25542 ) {
25543 let (mut rig, cache, mut engine, mut frame) = reduce_order_replay();
25544 cache
25545 .borrow_mut()
25546 .add_account(AccountAny::Margin(test_margin_account()))
25547 .unwrap();
25548 dispatch_reduce_frame(&rig, &frame, false);
25549 consume_reduce_events(&mut rig, &mut engine);
25550 let mut history = cache
25551 .borrow()
25552 .order(&rig.cloid)
25553 .unwrap()
25554 .events()
25555 .into_iter()
25556 .cloned()
25557 .collect::<Vec<_>>();
25558
25559 let OrderEventAny::Initialized(init) = &mut history[0] else {
25560 panic!("expected initialization");
25561 };
25562
25563 init.parent_order_id = Some(ClientOrderId::new("OFFLINE-PARENT"));
25564 let cached = OrderAny::from_events(history).unwrap();
25565 cache
25566 .borrow_mut()
25567 .add_order(cached.clone(), None, Some(client_id()), true)
25568 .unwrap();
25569
25570 let mut client = create_execution_client_with_cache(cache.clone());
25571 client.registry = rig.registry.clone();
25572 client.emitter = rig.emitter.clone();
25573 client.dispatch.seed_order_restorations(vec![cached]);
25574 client.dispatch.note_active_market(TEST_MARKET_INDEX);
25575 client.dispatch.replace_position_snapshot(&[], &[], &[]);
25576 frame.status = status;
25577
25578 let qty = Quantity::from(if status == LighterOrderStatus::Filled {
25579 "0.0027"
25580 } else {
25581 "0.0010"
25582 });
25583
25584 frame.filled_base_amount = qty.as_decimal();
25585 frame.filled_quote_amount = frame.filled_base_amount * frame.price;
25586 frame.remaining_base_amount = Decimal::ZERO;
25587 let mut trade = dispatcher_test_trade(&rig, true);
25588 trade.bid_id = frame.order_index;
25589 trade.bid_id_str = Some(frame.order_id.clone());
25590 trade.size = qty.as_decimal();
25591 trade.usd_amount = trade.size * trade.price;
25592 let trade_id = parse_lighter_trade_id(&trade).unwrap();
25593 let hits = Arc::new(AtomicUsize::new(0));
25594 let counted = hits.clone();
25595
25596 let venue = VenueOrderId::new(&frame.order_id);
25597
25598 let app = Router::new()
25599 .route(
25600 "/api/v1/accountActiveOrders",
25601 get(|| async { axum::Json(serde_json::json!({"code":200,"orders":[]})) }),
25602 )
25603 .route(
25604 "/api/v1/accountInactiveOrders",
25605 get(move || {
25606 let frame = frame.clone();
25607 async move { axum::Json(serde_json::json!({"code":200,"orders":[frame]})) }
25608 }),
25609 )
25610 .route(
25611 "/api/v1/trades",
25612 get(move |Query(query): Query<AHashMap<String, String>>| {
25613 let trade = trade.clone();
25614 let hits = counted.clone();
25615 async move {
25616 let trades = if query.contains_key("order_index") {
25617 hits.fetch_add(1, Ordering::Relaxed);
25618 vec![trade]
25619 } else {
25620 Vec::new()
25621 };
25622
25623 axum::Json(serde_json::json!({"code":200,"trades":trades}))
25624 }
25625 }),
25626 );
25627
25628 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
25629 let addr = listener.local_addr().unwrap();
25630 let server = tokio::spawn(async move {
25631 axum::serve(listener, app).await.unwrap();
25632 });
25633
25634 client.http_client = LighterHttpClient::new(
25635 LighterEnvironment::Testnet,
25636 Some(format!("http://{addr}")),
25637 1,
25638 None,
25639 )
25640 .unwrap();
25641 assert_eq!(
25642 cache.borrow().client_order_id(&venue).copied(),
25643 Some(rig.cloid)
25644 );
25645 let mass = client.generate_mass_status(None).await.unwrap().unwrap();
25646 assert!(
25647 client
25648 .dispatch
25649 .order_identity(&rig.cloid)
25650 .unwrap()
25651 .child_fill_baseline()
25652 .is_some()
25653 );
25654 wait_until_async(
25655 || async {
25656 client
25657 .dispatch
25658 .order_identity(&rig.cloid)
25659 .is_some_and(|identity| identity.has_child_publication())
25660 },
25661 Duration::from_secs(3),
25662 )
25663 .await;
25664
25665 assert!(mass.fill_reports().is_empty());
25666 client.pending_tasks.begin_shutdown();
25667 client
25668 .pending_tasks
25669 .finish_shutdown(Duration::from_secs(2), DISCONNECT_TIMEOUT)
25670 .await
25671 .unwrap();
25672
25673 for event in drain_events(&mut rig.rx) {
25674 match event {
25675 ExecutionEvent::Order(event) => engine.process(&event),
25676 ExecutionEvent::Report(report) => {
25677 engine.reconcile_execution_report(&report);
25678 }
25679 _ => panic!("unexpected child recovery event"),
25680 }
25681 }
25682
25683 let cached = cache.borrow().order(&rig.cloid).unwrap().clone();
25684 assert_eq!(hits.load(Ordering::Relaxed), 1);
25685 assert_eq!(
25686 cached.status(),
25687 if status == LighterOrderStatus::Filled {
25688 OrderStatus::Filled
25689 } else {
25690 OrderStatus::Canceled
25691 }
25692 );
25693 assert_eq!(cached.filled_qty(), qty);
25694 assert_eq!(cached.trade_ids(), vec![&trade_id]);
25695 assert!(
25696 inferred_reconciliation_trade_ids(&cached)
25697 .unwrap()
25698 .is_empty()
25699 );
25700 server.abort();
25701 let _ = server.await;
25702 }
25703
25704 #[tokio::test]
25705 async fn terminal_child_recovery_rechecks_inferred_overlap_after_http() {
25706 let (mut rig, cache, mut engine, mut frame) = reduce_order_replay();
25707 cache
25708 .borrow_mut()
25709 .add_account(AccountAny::Margin(test_margin_account()))
25710 .unwrap();
25711 dispatch_reduce_frame(&rig, &frame, false);
25712 consume_reduce_events(&mut rig, &mut engine);
25713 let mut history = cache
25714 .borrow()
25715 .order(&rig.cloid)
25716 .unwrap()
25717 .events()
25718 .into_iter()
25719 .cloned()
25720 .collect::<Vec<_>>();
25721
25722 let OrderEventAny::Initialized(init) = &mut history[0] else {
25723 panic!("expected initialization");
25724 };
25725
25726 init.parent_order_id = Some(ClientOrderId::new("RACING-PARENT"));
25727 let cached = OrderAny::from_events(history).unwrap();
25728 cache
25729 .borrow_mut()
25730 .add_order(cached.clone(), None, Some(client_id()), true)
25731 .unwrap();
25732 rig.dispatch = WsDispatchState::new();
25733 rig.dispatch
25734 .restore_reconciled_order(
25735 &cached,
25736 frame.client_order_index,
25737 VenueOrderId::new(&frame.order_id),
25738 false,
25739 )
25740 .unwrap();
25741 let identity = rig.dispatch.order_identity(&rig.cloid).unwrap();
25742
25743 let mut client = create_execution_client_with_cache(cache.clone());
25744 client.dispatch = rig.dispatch.clone();
25745 client.registry = rig.registry.clone();
25746 client.emitter = rig.emitter.clone();
25747 client.subscribe_applied_fills();
25748 frame.status = LighterOrderStatus::Canceled;
25749 frame.filled_base_amount = Decimal::from_str("0.0010").unwrap();
25750 frame.remaining_base_amount = Decimal::ZERO;
25751 frame.filled_quote_amount = frame.filled_base_amount * frame.price;
25752 let instrument = LIGHTER_INSTRUMENT_CACHE
25753 .get(&rig.instrument_id)
25754 .unwrap()
25755 .value()
25756 .clone();
25757 let mut partial =
25758 parse_ws_order_status_report(&frame, &instrument, account_id(), UnixNanos::from(3))
25759 .unwrap()
25760 .with_client_order_id(rig.cloid);
25761 partial.order_status = OrderStatus::PartiallyFilled;
25762 let mut trade = dispatcher_test_trade(&rig, true);
25763 trade.bid_id = frame.order_index;
25764 trade.bid_id_str = Some(frame.order_id.clone());
25765 trade.size = frame.filled_base_amount;
25766 trade.usd_amount = trade.size * trade.price;
25767 let requested = Arc::new(tokio::sync::Notify::new());
25768 let resume = Arc::new(tokio::sync::Notify::new());
25769 let signal = requested.clone();
25770 let gate = resume.clone();
25771
25772 let app = Router::new()
25773 .route(
25774 "/api/v1/accountActiveOrders",
25775 get(|| async { axum::Json(serde_json::json!({"code":200,"orders":[]})) }),
25776 )
25777 .route(
25778 "/api/v1/accountInactiveOrders",
25779 get(move || {
25780 let frame = frame.clone();
25781 async move { axum::Json(serde_json::json!({"code":200,"orders":[frame]})) }
25782 }),
25783 )
25784 .route(
25785 "/api/v1/trades",
25786 get(move || {
25787 let trade = trade.clone();
25788 let signal = signal.clone();
25789 let gate = gate.clone();
25790 async move {
25791 signal.notify_one();
25792 gate.notified().await;
25793 axum::Json(serde_json::json!({"code":200,"trades":[trade]}))
25794 }
25795 }),
25796 );
25797
25798 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
25799 let addr = listener.local_addr().unwrap();
25800 let server = tokio::spawn(async move {
25801 axum::serve(listener, app).await.unwrap();
25802 });
25803
25804 client.http_client = LighterHttpClient::new(
25805 LighterEnvironment::Testnet,
25806 Some(format!("http://{addr}")),
25807 1,
25808 None,
25809 )
25810 .unwrap();
25811 let context = acked_probe_context(&client);
25812 let (result, ()) = tokio::join!(recover_child_order(rig.cloid, 0, &context), async {
25813 tokio::time::timeout(Duration::from_secs(2), requested.notified())
25814 .await
25815 .unwrap();
25816 engine.reconcile_order_status_report(&partial);
25817 assert!(identity.has_inferred_fills());
25818 resume.notify_one();
25819 });
25820 assert!(result.unwrap());
25821 assert_eq!(
25822 client
25823 .dispatch
25824 .order_events
25825 .lock()
25826 .get(&rig.cloid)
25827 .map_or(0, |entry| entry.fills.len()),
25828 0
25829 );
25830 assert!(drain_events(&mut rig.rx).is_empty());
25831 assert_eq!(identity.filled_qty(), Decimal::ZERO);
25832 let cached = cache.borrow().order(&rig.cloid).unwrap().clone();
25833 let inferred = inferred_reconciliation_trade_ids(&cached).unwrap();
25834 assert_eq!(inferred.len(), 1);
25835 assert_eq!(cached.trade_ids(), inferred.iter().collect::<Vec<_>>());
25836 assert_eq!(cached.filled_qty(), Quantity::from("0.0010"));
25837 server.abort();
25838 let _ = server.await;
25839 }
25840
25841 fn create_execution_client_with_cache(cache: Rc<RefCell<Cache>>) -> LighterExecutionClient {
25842 let core = ExecutionClientCore::new(
25843 trader_id(),
25844 client_id(),
25845 test_config().resolved_venue(),
25846 OmsType::Netting,
25847 account_id(),
25848 AccountType::Margin,
25849 None,
25850 cache,
25851 );
25852 LighterExecutionClient::new(core, test_config()).unwrap()
25853 }
25854}