1use std::sync::{
33 Arc,
34 atomic::{AtomicBool, Ordering},
35};
36
37use ahash::AHashMap;
38use futures_util::{StreamExt, stream::FuturesUnordered};
39use nautilus_live::{
40 SocketControlFactory,
41 book::snapshot::SnapshotGate,
42 task::{TaskJoinOutcome, TaskSlot, TaskSpawnError, finish_task},
43};
44use nautilus_network::websocket::{TransportBackend, proxy::ProxyUrl};
45use parking_lot::Mutex;
46use ustr::Ustr;
47
48use super::{
49 MARKET_STREAMS_ENDPOINT,
50 client::{PolymarketWebSocketClient, WsSubscriptionHandle},
51 handler::CycleMarketOutcome,
52 messages::{MarketWsMessage, PolymarketWsMessage},
53};
54use crate::common::consts::WS_DEFAULT_SUBSCRIPTIONS;
55
56const PRIMARY_SHARD_ID: usize = 0;
58
59#[derive(Debug)]
61pub struct PolymarketMarketConnectionPool {
62 inner: Arc<PoolInner>,
63}
64
65#[derive(Clone, Debug)]
69pub struct PolymarketMarketPoolHandle {
70 inner: Arc<PoolInner>,
71}
72
73#[derive(Debug)]
74struct PoolInner {
75 base_url: Option<String>,
76 proxy_url: Option<ProxyUrl>,
77 transport_backend: TransportBackend,
78 subscribe_new_markets: bool,
79 max_subscriptions: usize,
80 connect_timeout_secs: Mutex<Option<u64>>,
81 wire_mutex: tokio::sync::Mutex<()>,
83 state: Mutex<PoolState>,
85 out_tx: Mutex<Option<tokio::sync::mpsc::UnboundedSender<PolymarketWsMessage>>>,
86 out_rx: Mutex<Option<tokio::sync::mpsc::UnboundedReceiver<PolymarketWsMessage>>>,
87 socket_factory: Mutex<Option<SocketControlFactory>>,
88 closed: AtomicBool,
89}
90
91#[derive(Debug, Default)]
92struct PoolState {
93 shards: AHashMap<usize, ShardEntry>,
94 assignments: AHashMap<Ustr, usize>,
95 shutdown_errors: Vec<String>,
96}
97
98struct PoolDrain<'a> {
99 owner: &'a Mutex<PoolState>,
100 state: PoolState,
101 completed: Vec<usize>,
102}
103
104impl<'a> PoolDrain<'a> {
105 fn take(owner: &'a Mutex<PoolState>) -> Self {
106 let state = std::mem::take(&mut *owner.lock());
107 Self {
108 owner,
109 state,
110 completed: Vec::new(),
111 }
112 }
113}
114
115impl Drop for PoolDrain<'_> {
116 fn drop(&mut self) {
117 for id in &self.completed {
118 self.state.shards.remove(id);
119 }
120
121 self.state
123 .assignments
124 .retain(|_, id| self.state.shards.contains_key(id));
125 *self.owner.lock() = std::mem::take(&mut self.state);
126 }
127}
128
129#[derive(Debug)]
130struct ShardEntry {
131 client: PolymarketWebSocketClient,
132 handle: WsSubscriptionHandle,
133 forwarder: TaskSlot<()>,
134 owned: usize,
135 closing: bool,
136}
137
138enum ReleaseOutcome {
139 NotOwned,
140 Unsubscribe(WsSubscriptionHandle),
141 CloseShard(usize, Box<ShardEntry>),
142}
143
144struct ShardClose<'a> {
145 owner: &'a Mutex<PoolState>,
146 id: usize,
147 shard: Option<Box<ShardEntry>>,
148}
149
150impl<'a> ShardClose<'a> {
151 fn new(owner: &'a Mutex<PoolState>, id: usize, mut shard: Box<ShardEntry>) -> Self {
152 shard.closing = true;
153 Self {
154 owner,
155 id,
156 shard: Some(shard),
157 }
158 }
159
160 fn shard_mut(&mut self) -> &mut ShardEntry {
161 self.shard.as_deref_mut().expect("closing shard present")
162 }
163
164 fn complete(mut self) {
165 self.shard.take();
166 }
167}
168
169impl Drop for ShardClose<'_> {
170 fn drop(&mut self) {
171 if let Some(shard) = self.shard.take() {
172 let replaced = self.owner.lock().shards.insert(self.id, *shard);
173 assert!(replaced.is_none(), "closing shard ID is already present");
174 }
175 }
176}
177
178#[allow(
179 clippy::missing_panics_doc,
180 reason = "internal mutex locks and shard-state invariants are not expected to panic"
181)]
182impl PolymarketMarketConnectionPool {
183 #[must_use]
188 pub fn new(
189 base_url: Option<String>,
190 subscribe_new_markets: bool,
191 transport_backend: TransportBackend,
192 max_subscriptions: usize,
193 ) -> Self {
194 Self::new_with_proxy(
195 base_url,
196 subscribe_new_markets,
197 transport_backend,
198 max_subscriptions,
199 None,
200 )
201 }
202
203 #[must_use]
205 pub fn new_with_proxy(
206 base_url: Option<String>,
207 subscribe_new_markets: bool,
208 transport_backend: TransportBackend,
209 max_subscriptions: usize,
210 proxy_url: Option<ProxyUrl>,
211 ) -> Self {
212 Self {
213 inner: Arc::new(PoolInner::new_with_proxy(
214 base_url,
215 transport_backend,
216 subscribe_new_markets,
217 max_subscriptions,
218 proxy_url,
219 )),
220 }
221 }
222
223 pub(crate) fn with_connect_timeout(self, timeout_secs: u64) -> Self {
224 *self.inner.connect_timeout_secs.lock() = Some(timeout_secs);
225 self
226 }
227
228 #[must_use]
230 pub(crate) fn with_socket_factory(self, factory: SocketControlFactory) -> Self {
231 *self.inner.socket_factory.lock() = Some(factory);
232 self
233 }
234
235 #[cfg(test)]
236 pub(crate) fn proxy_url(&self) -> Option<&ProxyUrl> {
237 self.inner.proxy_url.as_ref()
238 }
239
240 #[must_use]
242 pub fn handle(&self) -> PolymarketMarketPoolHandle {
243 PolymarketMarketPoolHandle {
244 inner: Arc::clone(&self.inner),
245 }
246 }
247
248 pub(crate) async fn prepare(&self) -> anyhow::Result<()> {
249 if self.inner.closed.load(Ordering::Acquire) {
250 self.disconnect().await?;
251 }
252
253 let _wire = self.inner.wire_mutex.lock().await;
254 self.inner.prepare_channel();
255 Ok(())
256 }
257
258 pub async fn connect(&self) -> anyhow::Result<()> {
264 if self.inner.closed.load(Ordering::Acquire) {
265 self.disconnect().await?;
266 }
267
268 let _wire = self.inner.wire_mutex.lock().await;
269
270 if !self.inner.closed.load(Ordering::Acquire) && !self.inner.state.lock().shards.is_empty()
271 {
272 log::warn!("Polymarket market pool already connected");
273 return Ok(());
274 }
275
276 self.inner.prepare_channel();
277 self.inner.connect_new_shard(true).await?;
278 Ok(())
279 }
280
281 pub async fn subscribe_new_markets_feed(&self) -> anyhow::Result<()> {
287 let _wire = self.inner.wire_mutex.lock().await;
288
289 let handle = {
290 let state = self.inner.state.lock();
291 if self.inner.closed.load(Ordering::Acquire) {
292 anyhow::bail!("Market connection pool is closed");
293 }
294 state
295 .shards
296 .get(&PRIMARY_SHARD_ID)
297 .map(|shard| shard.handle.clone())
298 };
299
300 match handle {
301 Some(handle) => handle.subscribe_market(vec![]).await,
302 None => anyhow::bail!("No primary market shard available for new-market discovery"),
303 }
304 }
305
306 #[must_use]
308 pub fn take_message_receiver(
309 &self,
310 ) -> Option<tokio::sync::mpsc::UnboundedReceiver<PolymarketWsMessage>> {
311 self.inner.out_rx.lock().take()
312 }
313
314 pub async fn disconnect(&self) -> anyhow::Result<()> {
321 self.inner.begin_shutdown();
322 let _wire = self.inner.wire_mutex.lock().await;
323
324 let mut drain = PoolDrain::take(&self.inner.state);
325 {
326 let errors = Mutex::new(&mut drain.state.shutdown_errors);
327 let mut disconnects = drain
328 .state
329 .shards
330 .iter_mut()
331 .map(|(&shard_id, shard)| {
332 let errors = &errors;
333 async move {
334 let mut failed = false;
335 shard.forwarder.abort();
336 if let Some(outcome) = finish_task(
337 &mut shard.forwarder,
338 std::time::Duration::ZERO,
339 std::time::Duration::from_secs(2),
340 )
341 .await
342 {
343 match outcome {
344 TaskJoinOutcome::Completed(()) | TaskJoinOutcome::Aborted => {}
345 TaskJoinOutcome::Failed(error) => {
346 failed = true;
347 errors.lock().push(format!(
348 "market shard {shard_id} forwarder failed: {error}"
349 ));
350 }
351 TaskJoinOutcome::Incomplete => {
352 failed = true;
353 errors.lock().push(format!(
354 "market shard {shard_id} forwarder did not stop after abort"
355 ));
356 }
357 }
358 }
359
360 if let Err(e) = shard.client.disconnect().await {
361 failed = true;
362 errors
363 .lock()
364 .push(format!("market shard {shard_id} disconnect failed: {e}"));
365 }
366 (shard_id, failed)
367 }
368 })
369 .collect::<FuturesUnordered<_>>();
370
371 while let Some((shard_id, failed)) = disconnects.next().await {
372 if !failed {
373 drain.completed.push(shard_id);
374 }
375 }
376 }
377
378 if !drain.state.shutdown_errors.is_empty() {
379 let errors = std::mem::take(&mut drain.state.shutdown_errors);
380 anyhow::bail!(
381 "Polymarket market pool shutdown failed: {}",
382 errors.join("; ")
383 );
384 }
385
386 *self.inner.out_tx.lock() = None;
387 *self.inner.out_rx.lock() = None;
388 Ok(())
389 }
390
391 pub(crate) fn begin_shutdown(&self) {
392 self.inner.begin_shutdown();
393 }
394
395 pub(crate) fn clear_reconnect_state(&self) {
397 let state = self.inner.state.lock();
398 for shard in state.shards.values() {
399 shard.client.clear_reconnect_state();
400 }
401 }
402
403 #[must_use]
405 pub fn connection_count(&self) -> usize {
406 self.inner.state.lock().shards.len()
407 }
408
409 #[must_use]
411 pub fn subscription_count(&self) -> usize {
412 self.inner.state.lock().assignments.len()
413 }
414}
415
416impl Drop for PoolInner {
417 fn drop(&mut self) {
418 self.closed.store(true, Ordering::Release);
419
420 for shard in self.state.get_mut().shards.values_mut() {
421 shard.forwarder.abort();
422 shard.client.abort();
423 }
424 }
425}
426
427#[allow(
428 clippy::missing_panics_doc,
429 reason = "internal mutex locks and shard-state invariants are not expected to panic"
430)]
431impl PolymarketMarketPoolHandle {
432 pub(crate) fn begin_shutdown(&self) {
433 self.inner.begin_shutdown();
434 }
435
436 pub async fn subscribe_market(&self, asset_ids: Vec<String>) -> anyhow::Result<()> {
442 let _wire = self.inner.wire_mutex.lock().await;
443 self.inner.ensure_open()?;
444 for asset_id in asset_ids {
445 self.inner.subscribe_one(asset_id).await?;
446 }
447 Ok(())
448 }
449
450 pub async fn unsubscribe_market(&self, asset_ids: Vec<String>) -> anyhow::Result<()> {
456 let _wire = self.inner.wire_mutex.lock().await;
457 self.inner.ensure_open()?;
458 for asset_id in asset_ids {
459 self.inner.unsubscribe_one(asset_id).await?;
460 }
461 Ok(())
462 }
463
464 pub async fn resubscribe_market(
476 &self,
477 asset_ids: Vec<String>,
478 cancel: &tokio_util::sync::CancellationToken,
479 gate: &SnapshotGate,
480 ) -> anyhow::Result<CycleMarketOutcome> {
481 let mut outcome = CycleMarketOutcome::Completed;
482
483 for asset_id in asset_ids {
484 outcome = self.inner.cycle_one(&asset_id, cancel, gate).await?;
485
486 if !matches!(outcome, CycleMarketOutcome::Completed) {
487 break;
488 }
489 }
490
491 Ok(outcome)
492 }
493
494 pub(crate) fn tokens_for_shard(&self, shard_id: usize) -> Vec<Ustr> {
496 let state = self.inner.state.lock();
497 state
498 .assignments
499 .iter()
500 .filter_map(|(token, assigned)| (*assigned == shard_id).then_some(*token))
501 .collect()
502 }
503}
504
505impl PoolInner {
506 fn begin_shutdown(&self) {
507 let state = self.state.lock();
508 self.closed.store(true, Ordering::Release);
509
510 for shard in state.shards.values() {
511 shard.client.begin_shutdown();
512 }
513 }
514
515 fn prepare_channel(&self) {
516 {
517 let _state = self.state.lock();
518 self.closed.store(false, Ordering::Release);
519 }
520
521 if self.out_tx.lock().is_some() {
522 return;
523 }
524
525 let (out_tx, out_rx) = tokio::sync::mpsc::unbounded_channel();
526 *self.out_tx.lock() = Some(out_tx);
527 *self.out_rx.lock() = Some(out_rx);
528 }
529
530 fn ensure_open(&self) -> anyhow::Result<()> {
531 let _state = self.state.lock();
532
533 if self.closed.load(Ordering::Acquire) {
534 anyhow::bail!("Market connection pool is closed");
535 }
536 Ok(())
537 }
538}
539
540impl PoolInner {
541 #[cfg(test)]
542 fn new(
543 base_url: Option<String>,
544 transport_backend: TransportBackend,
545 subscribe_new_markets: bool,
546 max_subscriptions: usize,
547 ) -> Self {
548 Self::new_with_proxy(
549 base_url,
550 transport_backend,
551 subscribe_new_markets,
552 max_subscriptions,
553 None,
554 )
555 }
556
557 fn new_with_proxy(
558 base_url: Option<String>,
559 transport_backend: TransportBackend,
560 subscribe_new_markets: bool,
561 max_subscriptions: usize,
562 proxy_url: Option<ProxyUrl>,
563 ) -> Self {
564 let max_subscriptions = if max_subscriptions == 0 {
565 log::warn!(
566 "PolymarketDataClientConfig.ws_max_subscriptions=0 is invalid, using {WS_DEFAULT_SUBSCRIPTIONS}"
567 );
568 WS_DEFAULT_SUBSCRIPTIONS
569 } else {
570 max_subscriptions
571 };
572
573 Self {
574 base_url,
575 proxy_url,
576 transport_backend,
577 subscribe_new_markets,
578 max_subscriptions,
579 connect_timeout_secs: Mutex::new(None),
580 wire_mutex: tokio::sync::Mutex::new(()),
581 state: Mutex::new(PoolState::default()),
582 out_tx: Mutex::new(None),
583 out_rx: Mutex::new(None),
584 socket_factory: Mutex::new(None),
585 closed: AtomicBool::new(false),
586 }
587 }
588
589 async fn subscribe_one(&self, asset_id: String) -> anyhow::Result<()> {
591 let token = Ustr::from(asset_id.as_str());
592
593 let Some(handle) = self.assign(token).await? else {
594 return Ok(());
595 };
596
597 let result = match self.ensure_open() {
598 Ok(()) => handle.subscribe_market(vec![asset_id]).await,
599 Err(e) => Err(e),
600 };
601
602 if let Err(e) = result {
603 if let ReleaseOutcome::CloseShard(id, shard) = self.release(token)
605 && let Err(close_error) = self.close_shard(id, shard).await
606 {
607 anyhow::bail!("{e}; subscription rollback failed: {close_error}");
608 }
609 return Err(e);
610 }
611 Ok(())
612 }
613
614 async fn unsubscribe_one(&self, asset_id: String) -> anyhow::Result<()> {
616 self.ensure_open()?;
617 let token = Ustr::from(asset_id.as_str());
618
619 match self.release(token) {
620 ReleaseOutcome::NotOwned => Ok(()),
621 ReleaseOutcome::Unsubscribe(handle) => handle.unsubscribe_market(vec![asset_id]).await,
622 ReleaseOutcome::CloseShard(id, shard) => {
623 self.close_shard(id, shard).await
625 }
626 }
627 }
628
629 async fn cycle_one(
630 &self,
631 asset_id: &str,
632 cancel: &tokio_util::sync::CancellationToken,
633 gate: &SnapshotGate,
634 ) -> anyhow::Result<CycleMarketOutcome> {
635 let responder_rx = {
636 let _wire = self.wire_mutex.lock().await;
637 self.ensure_open()?;
638
639 let handle = {
640 let state = self.state.lock();
641 let token = Ustr::from(asset_id);
642
643 let Some(id) = state.assignments.get(&token).copied() else {
644 anyhow::bail!("Market asset {asset_id} is not owned by any shard");
645 };
646
647 state
648 .shards
649 .get(&id)
650 .map(|shard| shard.handle.clone())
651 .ok_or_else(|| anyhow::anyhow!("Market shard {id} is not available"))?
652 };
653
654 let (responder_tx, responder_rx) = tokio::sync::oneshot::channel();
657 handle
658 .cycle_market_subscription(
659 vec![asset_id.to_string()],
660 cancel.clone(),
661 responder_tx,
662 gate.clone(),
663 )
664 .await?;
665 responder_rx
666 };
667
668 responder_rx
669 .await
670 .map_err(|_| anyhow::anyhow!("Market subscription cycle response lost"))
671 }
672
673 async fn assign(&self, token: Ustr) -> anyhow::Result<Option<WsSubscriptionHandle>> {
675 {
676 let mut state = self.state.lock();
677
678 if self.closed.load(Ordering::Acquire) {
679 anyhow::bail!("Market connection pool is closed");
680 }
681
682 if state.assignments.contains_key(&token) {
683 return Ok(None);
684 }
685
686 if let Some(id) = smallest_shard_with_capacity(&state, self.max_subscriptions) {
687 let handle = {
688 let shard = state.shards.get_mut(&id).expect("shard present");
689 shard.owned += 1;
690 shard.handle.clone()
691 };
692 state.assignments.insert(token, id);
693 return Ok(Some(handle));
694 }
695 }
696
697 let id = self.connect_new_shard(false).await?;
698
699 let rejected_shard = {
700 let mut state = self.state.lock();
701
702 if !self.closed.load(Ordering::Acquire) {
703 let handle = {
704 let shard = state.shards.get_mut(&id).expect("new shard present");
705 shard.owned += 1;
706 shard.handle.clone()
707 };
708 state.assignments.insert(token, id);
709 return Ok(Some(handle));
710 }
711
712 state
713 .shards
714 .remove(&id)
715 .expect("new shard retained for shutdown")
716 };
717
718 if let Err(e) = self.close_shard(id, Box::new(rejected_shard)).await {
719 anyhow::bail!("Market connection pool is closed; shard rollback failed: {e}");
720 }
721
722 anyhow::bail!("Market connection pool is closed")
723 }
724
725 fn release(&self, token: Ustr) -> ReleaseOutcome {
726 let mut state = self.state.lock();
727
728 let Some(id) = state.assignments.remove(&token) else {
729 return ReleaseOutcome::NotOwned;
730 };
731
732 let owned = {
733 let Some(shard) = state.shards.get_mut(&id) else {
734 return ReleaseOutcome::NotOwned;
735 };
736 shard.owned = shard.owned.saturating_sub(1);
737 shard.owned
738 };
739
740 if id != PRIMARY_SHARD_ID && owned == 0 {
741 let shard = state.shards.remove(&id).expect("shard present");
742 ReleaseOutcome::CloseShard(id, Box::new(shard))
743 } else {
744 let handle = state.shards.get(&id).expect("shard present").handle.clone();
745 ReleaseOutcome::Unsubscribe(handle)
746 }
747 }
748
749 async fn connect_new_shard(&self, is_primary: bool) -> anyhow::Result<usize> {
750 if self.closed.load(Ordering::Acquire) {
751 anyhow::bail!("Market connection pool is closed");
752 }
753
754 let id = if is_primary {
755 PRIMARY_SHARD_ID
756 } else {
757 let state = self.state.lock();
758 available_shard_id(&state)
759 };
760
761 let mut client = self.market_client(self.subscribe_new_markets, id);
762 client.connect().await?;
763
764 let handle = client.clone_subscription_handle();
765 let rx = client
766 .take_message_receiver()
767 .ok_or_else(|| anyhow::anyhow!("Market shard receiver unavailable after connect"))?;
768
769 let forwarder = match self.spawn_forwarder(rx, id) {
770 Ok(forwarder) => forwarder,
771 Err((e, forwarder)) => {
772 let shard = Box::new(ShardEntry {
773 client,
774 handle,
775 forwarder,
776 owned: 0,
777 closing: true,
778 });
779
780 if let Err(close_error) = self.close_shard(id, shard).await {
781 anyhow::bail!(
782 "Failed to start market shard forwarder: {e}; startup rollback failed: \
783 {close_error}"
784 );
785 }
786 anyhow::bail!("Failed to start market shard forwarder: {e}");
787 }
788 };
789
790 let shard = ShardEntry {
791 client,
792 handle,
793 forwarder,
794 owned: 0,
795 closing: false,
796 };
797 let rejected_shard = {
798 let mut state = self.state.lock();
799
800 if self.closed.load(Ordering::Acquire) {
801 Some(shard)
802 } else {
803 state.shards.insert(id, shard);
804 None
805 }
806 };
807
808 if let Some(shard) = rejected_shard {
809 if let Err(e) = self.close_shard(id, Box::new(shard)).await {
810 anyhow::bail!("Market connection pool is closed; shard rollback failed: {e}");
811 }
812 anyhow::bail!("Market connection pool is closed");
813 }
814
815 log::debug!("Opened Polymarket market shard {id}");
816 Ok(id)
817 }
818
819 fn market_client(
820 &self,
821 subscribe_new_markets: bool,
822 shard_id: usize,
823 ) -> PolymarketWebSocketClient {
824 let client = PolymarketWebSocketClient::new_market_with_proxy(
825 self.base_url.clone(),
826 subscribe_new_markets,
827 self.transport_backend,
828 self.proxy_url.clone(),
829 );
830
831 let client = match *self.connect_timeout_secs.lock() {
832 Some(timeout_secs) => client.with_connect_timeout(timeout_secs),
833 None => client,
834 };
835
836 let factory = self.socket_factory.lock().clone();
837
838 if let Some(factory) = factory {
839 let endpoint = if shard_id == PRIMARY_SHARD_ID {
840 MARKET_STREAMS_ENDPOINT.to_string()
841 } else {
842 format!("{MARKET_STREAMS_ENDPOINT}-{shard_id}")
843 };
844 client.with_socket_control(factory.control(endpoint))
845 } else {
846 client
847 }
848 }
849
850 fn spawn_forwarder(
851 &self,
852 mut rx: tokio::sync::mpsc::UnboundedReceiver<PolymarketWsMessage>,
853 shard_id: usize,
854 ) -> Result<TaskSlot<()>, (TaskSpawnError, TaskSlot<()>)> {
855 let out_tx = self.out_tx.lock().clone();
856 let is_primary = shard_id == PRIMARY_SHARD_ID;
857
858 let mut forwarder = TaskSlot::new();
859 if let Err(e) = forwarder.spawn(async move {
860 let Some(out_tx) = out_tx else {
861 return;
862 };
863
864 while let Some(msg) = rx.recv().await {
865 if !should_forward_from_shard(&msg, is_primary) {
866 continue;
867 }
868
869 let msg = match msg {
870 PolymarketWsMessage::Reconnected { .. } => PolymarketWsMessage::Reconnected {
871 shard_id: Some(shard_id),
872 },
873 other => other,
874 };
875
876 if out_tx.send(msg).is_err() {
877 break;
878 }
879 }
880 }) {
881 return Err((e, forwarder));
882 }
883 Ok(forwarder)
884 }
885
886 async fn close_shard(&self, id: usize, shard: Box<ShardEntry>) -> anyhow::Result<()> {
887 let mut close = ShardClose::new(&self.state, id, shard);
888 close.shard_mut().forwarder.abort();
889 let forwarder_stopped = match finish_task(
890 &mut close.shard_mut().forwarder,
891 std::time::Duration::ZERO,
892 std::time::Duration::from_secs(2),
893 )
894 .await
895 {
896 None | Some(TaskJoinOutcome::Completed(()) | TaskJoinOutcome::Aborted) => true,
897 Some(TaskJoinOutcome::Failed(error)) => {
898 let error = format!("market shard {id} forwarder failed: {error}");
899 self.state.lock().shutdown_errors.push(error);
900 true
901 }
902 Some(TaskJoinOutcome::Incomplete) => {
903 let error = format!("market shard {id} forwarder did not stop after abort");
904 self.state.lock().shutdown_errors.push(error);
905 false
906 }
907 };
908
909 if let Err(e) = close.shard_mut().client.disconnect().await {
910 let error = format!("market shard {id} disconnect failed: {e}");
911 self.state.lock().shutdown_errors.push(error);
912 }
913
914 if forwarder_stopped && !close.shard_mut().client.has_task() {
915 close.complete();
916 }
917
918 let errors = std::mem::take(&mut self.state.lock().shutdown_errors);
919
920 if errors.is_empty() {
921 Ok(())
922 } else {
923 anyhow::bail!(errors.join("; "))
924 }
925 }
926
927 #[cfg(test)]
928 fn subscription_count_for_test(&self) -> usize {
929 self.state.lock().assignments.len()
930 }
931}
932
933fn should_forward_from_shard(message: &PolymarketWsMessage, is_primary: bool) -> bool {
934 is_primary
935 || !matches!(
936 message,
937 PolymarketWsMessage::Market(
938 MarketWsMessage::NewMarket(_) | MarketWsMessage::MarketResolved(_)
939 )
940 )
941}
942
943fn smallest_shard_with_capacity(state: &PoolState, max_subscriptions: usize) -> Option<usize> {
944 state
945 .shards
946 .iter()
947 .filter(|(_, shard)| !shard.closing && shard.owned < max_subscriptions)
948 .map(|(id, _)| *id)
949 .min()
950}
951
952fn available_shard_id(state: &PoolState) -> usize {
953 let mut id = PRIMARY_SHARD_ID + 1;
954 while state.shards.contains_key(&id) {
955 id = id.checked_add(1).expect("market shard ID space exhausted");
956 }
957 id
958}
959
960#[cfg(test)]
961impl PolymarketMarketPoolHandle {
962 pub(crate) fn test_single_shard(
965 sender: tokio::sync::mpsc::UnboundedSender<super::handler::HandlerCommand>,
966 assigned: &[&str],
967 ) -> Self {
968 let inner = PoolInner::new(
969 None,
970 TransportBackend::default(),
971 false,
972 WS_DEFAULT_SUBSCRIPTIONS,
973 );
974 {
975 let mut state = inner.state.lock();
976 state.shards.insert(
977 PRIMARY_SHARD_ID,
978 ShardEntry {
979 client: PolymarketWebSocketClient::new_market(
980 None,
981 false,
982 TransportBackend::default(),
983 ),
984 handle: WsSubscriptionHandle::from_sender(sender),
985 forwarder: TaskSlot::new(),
986 owned: assigned.len(),
987 closing: false,
988 },
989 );
990
991 for token in assigned {
992 state
993 .assignments
994 .insert(Ustr::from(token), PRIMARY_SHARD_ID);
995 }
996 }
997 Self {
998 inner: Arc::new(inner),
999 }
1000 }
1001}
1002
1003#[cfg(test)]
1004mod tests {
1005 use std::{sync::Arc as StdArc, time::Duration};
1006
1007 use PolymarketMarketPoolHandle as Handle;
1008 use nautilus_network::transport::TransportError;
1009 use parking_lot::{Condvar, Mutex as TestMutex};
1010 use rstest::rstest;
1011 use tokio_util::sync::CancellationToken;
1012
1013 use super::*;
1014 use crate::websocket::handler::HandlerCommand;
1015
1016 struct BlockingDrop(StdArc<(TestMutex<(bool, bool)>, Condvar)>);
1017
1018 impl Drop for BlockingDrop {
1019 fn drop(&mut self) {
1020 let (state, wake) = &*self.0;
1021 let mut state = state.lock();
1022 state.0 = true;
1023 wake.notify_all();
1024 while !state.1 {
1025 wake.wait(&mut state);
1026 }
1027 }
1028 }
1029
1030 fn state_with_shards(owned: &[usize]) -> PoolState {
1032 let mut state = PoolState::default();
1033
1034 for (id, owned) in owned.iter().enumerate() {
1035 let (tx, _rx) = tokio::sync::mpsc::unbounded_channel();
1036 state.shards.insert(
1037 id,
1038 ShardEntry {
1039 client: PolymarketWebSocketClient::new_market(
1040 None,
1041 false,
1042 TransportBackend::default(),
1043 ),
1044 handle: WsSubscriptionHandle::from_sender(tx),
1045 forwarder: TaskSlot::new(),
1046 owned: *owned,
1047 closing: false,
1048 },
1049 );
1050 }
1051 state
1052 }
1053
1054 fn market_message(filename: &str) -> PolymarketWsMessage {
1055 let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
1056 .join("test_data")
1057 .join(filename);
1058 let json = std::fs::read_to_string(path).unwrap();
1059 PolymarketWsMessage::Market(serde_json::from_str(&json).unwrap())
1060 }
1061
1062 #[rstest]
1063 #[case::primary_new_market("ws_market_new_market_msg.json", true, true)]
1064 #[case::secondary_new_market("ws_market_new_market_msg.json", false, false)]
1065 #[case::primary_resolution("ws_market_resolved_msg.json", true, true)]
1066 #[case::secondary_resolution("ws_market_resolved_msg.json", false, false)]
1067 #[case::secondary_best_bid_ask("ws_market_best_bid_ask_msg.json", false, true)]
1068 fn shard_forwarding_keeps_global_events_on_primary(
1069 #[case] filename: &str,
1070 #[case] is_primary: bool,
1071 #[case] expected: bool,
1072 ) {
1073 let message = market_message(filename);
1074 assert_eq!(should_forward_from_shard(&message, is_primary), expected);
1075 }
1076
1077 #[rstest]
1078 #[tokio::test]
1079 async fn secondary_forwarder_drops_global_events_and_keeps_best_bid_ask() {
1080 let inner = PoolInner::new(None, TransportBackend::default(), true, 1);
1081 let (out_tx, mut out_rx) = tokio::sync::mpsc::unbounded_channel();
1082 *inner.out_tx.lock() = Some(out_tx);
1083 let (shard_tx, shard_rx) = tokio::sync::mpsc::unbounded_channel();
1084 let mut forwarder = inner
1085 .spawn_forwarder(shard_rx, PRIMARY_SHARD_ID + 1)
1086 .expect("spawn forwarder");
1087
1088 shard_tx
1089 .send(market_message("ws_market_new_market_msg.json"))
1090 .unwrap();
1091 shard_tx
1092 .send(market_message("ws_market_resolved_msg.json"))
1093 .unwrap();
1094 shard_tx
1095 .send(market_message("ws_market_best_bid_ask_msg.json"))
1096 .unwrap();
1097 drop(shard_tx);
1098 let outcome = finish_task(
1099 &mut forwarder,
1100 Duration::from_secs(1),
1101 Duration::from_secs(1),
1102 )
1103 .await
1104 .expect("forwarder task");
1105 assert!(matches!(outcome, TaskJoinOutcome::Completed(())));
1106
1107 let forwarded = out_rx.try_recv().unwrap();
1108 let PolymarketWsMessage::Market(MarketWsMessage::BestBidAsk(message)) = forwarded else {
1109 panic!("unexpected forwarded message: {forwarded:?}");
1110 };
1111 assert_eq!(
1112 message.asset_id,
1113 Ustr::from(
1114 "85354956062430465315924116860125388538595433819574542752031640332592237464430"
1115 ),
1116 );
1117 assert!(out_rx.try_recv().is_err());
1118 }
1119
1120 #[rstest]
1121 #[tokio::test]
1122 async fn pool_drain_restores_remaining_state_on_drop() {
1123 let (cmd_tx, _cmd_rx) = tokio::sync::mpsc::unbounded_channel();
1124 let handle = Handle::test_single_shard(cmd_tx, &["token-a"]);
1125 handle
1126 .inner
1127 .state
1128 .lock()
1129 .shutdown_errors
1130 .push("before drain".to_string());
1131
1132 {
1133 let mut drain = PoolDrain::take(&handle.inner.state);
1134 let owner = handle.inner.state.lock();
1135 assert!(owner.shards.is_empty());
1136 assert!(owner.assignments.is_empty());
1137 assert!(owner.shutdown_errors.is_empty());
1138 drop(owner);
1139
1140 assert_eq!(drain.state.shards.len(), 1);
1141 assert_eq!(
1142 drain.state.assignments,
1143 AHashMap::from([(Ustr::from("token-a"), PRIMARY_SHARD_ID)]),
1144 );
1145 assert_eq!(drain.state.shutdown_errors, vec!["before drain"]);
1146 drain.state.shutdown_errors.push("during drain".to_string());
1147 }
1148
1149 let restored = handle.inner.state.lock();
1150 assert_eq!(restored.shards.len(), 1);
1151 let shard = restored
1152 .shards
1153 .get(&PRIMARY_SHARD_ID)
1154 .expect("restored shard");
1155 assert_eq!(shard.owned, 1);
1156 assert!(!shard.closing);
1157 assert_eq!(
1158 restored.assignments,
1159 AHashMap::from([(Ustr::from("token-a"), PRIMARY_SHARD_ID)]),
1160 );
1161 assert_eq!(
1162 restored.shutdown_errors,
1163 vec!["before drain", "during drain"]
1164 );
1165 }
1166
1167 async fn blocked_forwarder() -> (TaskSlot<()>, std::sync::mpsc::Sender<()>) {
1168 let (release, blocked) = std::sync::mpsc::channel();
1169 let (started, ready) = tokio::sync::oneshot::channel();
1170 let handle = tokio::task::spawn_blocking(move || {
1171 let _ = started.send(());
1172 let _ = blocked.recv();
1173 });
1174 ready.await.unwrap();
1175 (TaskSlot::from_handle(handle), release)
1176 }
1177
1178 #[rstest]
1179 #[case::single(1)]
1180 #[case::many(64)]
1181 #[tokio::test(start_paused = true)]
1182 async fn pool_disconnects_concurrently_and_preserves_failures(#[case] count: usize) {
1183 let pool = PolymarketMarketConnectionPool::new(None, false, TransportBackend::default(), 1);
1184 let mut state = state_with_shards(&vec![1; count]);
1185 let mut releases = Vec::new();
1186
1187 for (&id, shard) in &mut state.shards {
1188 let (forwarder, release) = blocked_forwarder().await;
1189 shard.forwarder = forwarder;
1190 releases.push(release);
1191 state.assignments.insert(Ustr::from(&id.to_string()), id);
1192 }
1193 *pool.inner.state.lock() = state;
1194
1195 let started = tokio::time::Instant::now();
1196 let expected = Duration::from_secs(2);
1197 let mut disconnect = std::pin::pin!(pool.disconnect());
1198 let result = loop {
1199 if let std::task::Poll::Ready(result) = futures_util::poll!(disconnect.as_mut()) {
1200 break result;
1201 }
1202 assert!(started.elapsed() < expected + Duration::from_secs(1));
1203 tokio::time::advance(Duration::from_millis(100)).await;
1204 };
1205 assert!(started.elapsed() >= expected);
1206 let error = result.unwrap_err().to_string();
1207 let failures = error.matches("forwarder did not stop after abort");
1208 assert_eq!(failures.count(), count);
1209 assert_eq!(pool.connection_count(), count);
1210 assert_eq!(pool.subscription_count(), count);
1211
1212 drop(releases);
1213 pool.disconnect().await.expect("cleanup retry");
1214 assert_eq!(pool.connection_count(), 0);
1215 assert_eq!(pool.subscription_count(), 0);
1216 }
1217
1218 #[rstest]
1219 #[tokio::test]
1220 async fn canceled_pool_disconnect_restores_only_unfinished_ownership() {
1221 let pool = PolymarketMarketConnectionPool::new(None, false, TransportBackend::default(), 1);
1222 let mut state = state_with_shards(&[1, 1]);
1223 let (forwarder, release) = blocked_forwarder().await;
1224 state.shards.get_mut(&1).unwrap().forwarder = forwarder;
1225 state.assignments = AHashMap::from([(Ustr::from("done"), 0), (Ustr::from("pending"), 1)]);
1226 *pool.inner.state.lock() = state;
1227
1228 {
1229 let mut disconnect = std::pin::pin!(pool.disconnect());
1230 assert!(futures_util::poll!(disconnect.as_mut()).is_pending());
1231 }
1232 assert_eq!(pool.connection_count(), 1);
1233 assert_eq!(
1234 pool.inner.state.lock().assignments,
1235 AHashMap::from([(Ustr::from("pending"), 1)])
1236 );
1237
1238 drop(release);
1239 pool.disconnect().await.expect("cleanup retry");
1240 assert_eq!(pool.connection_count(), 0);
1241 assert_eq!(pool.subscription_count(), 0);
1242 }
1243
1244 #[rstest]
1245 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1246 async fn canceled_shard_close_restores_unfinished_ownership() {
1247 let inner = Arc::new(PoolInner::new(None, TransportBackend::default(), false, 1));
1248 let blocking = StdArc::new((TestMutex::new((false, false)), Condvar::new()));
1249 let blocking_task = StdArc::clone(&blocking);
1250 let (started_tx, started_rx) = tokio::sync::oneshot::channel();
1251 let mut forwarder = TaskSlot::new();
1252 forwarder
1253 .spawn(async move {
1254 let _blocking = BlockingDrop(blocking_task);
1255 let _ = started_tx.send(());
1256 std::future::pending::<()>().await;
1257 })
1258 .expect("spawn forwarder");
1259 started_rx.await.expect("forwarder should start");
1260
1261 let (cmd_tx, _cmd_rx) = tokio::sync::mpsc::unbounded_channel();
1262 let shard = Box::new(ShardEntry {
1263 client: PolymarketWebSocketClient::new_market(None, false, TransportBackend::default()),
1264 handle: WsSubscriptionHandle::from_sender(cmd_tx),
1265 forwarder,
1266 owned: 0,
1267 closing: false,
1268 });
1269 let close_inner = Arc::clone(&inner);
1270 let close = tokio::spawn(async move {
1271 let _result = close_inner.close_shard(1, shard).await;
1272 });
1273
1274 loop {
1275 if blocking.0.lock().0 {
1276 break;
1277 }
1278 tokio::task::yield_now().await;
1279 }
1280 close.abort();
1281 let _ = close.await;
1282
1283 {
1284 let restored = inner.state.lock();
1285 let shard = restored.shards.get(&1).expect("closing shard restored");
1286 assert!(shard.closing);
1287 assert!(shard.forwarder.is_some());
1288 assert_eq!(smallest_shard_with_capacity(&restored, 1), None);
1289 }
1290
1291 {
1292 let (state, wake) = &*blocking;
1293 state.lock().1 = true;
1294 wake.notify_all();
1295 }
1296 let shard = inner
1297 .state
1298 .lock()
1299 .shards
1300 .remove(&1)
1301 .expect("restored shard");
1302 inner
1303 .close_shard(1, Box::new(shard))
1304 .await
1305 .expect("restored shard should close");
1306
1307 assert!(inner.state.lock().shards.is_empty());
1308 }
1309
1310 #[rstest]
1311 fn zero_max_subscriptions_clamps_to_default() {
1312 let inner = PoolInner::new(None, TransportBackend::default(), false, 0);
1313 assert_eq!(inner.max_subscriptions, WS_DEFAULT_SUBSCRIPTIONS);
1314 }
1315
1316 #[rstest]
1317 #[tokio::test]
1318 async fn secondary_market_connection_timeout() {
1319 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
1320 let url = format!("ws://{}", listener.local_addr().unwrap());
1321
1322 let server = tokio::spawn(async move {
1323 let mut connections = Vec::new();
1324
1325 loop {
1326 let (stream, _) = listener.accept().await.unwrap();
1327 connections.push(stream);
1328 }
1329 });
1330
1331 let pool =
1332 PolymarketMarketConnectionPool::new(Some(url), false, TransportBackend::default(), 1)
1333 .with_connect_timeout(1);
1334 let mut ws_client = pool.inner.market_client(false, PRIMARY_SHARD_ID + 1);
1335
1336 let error = tokio::time::timeout(Duration::from_secs(20), ws_client.connect())
1337 .await
1338 .expect("secondary shard must use the configured connection timeout")
1339 .unwrap_err();
1340 server.abort();
1341
1342 assert!(
1343 matches!(error.downcast_ref::<TransportError>(),
1344 Some(TransportError::Io(e)) if e.kind() == std::io::ErrorKind::TimedOut),
1345 "{error:#}",
1346 );
1347 }
1348
1349 #[rstest]
1350 fn pool_retains_proxy_for_lazily_created_shards() {
1351 const PROXY_URL: &str = "http://pool-user:pool-proxy-secret@127.0.0.1:18088";
1352 let pool = PolymarketMarketConnectionPool::new_with_proxy(
1353 Some("ws://market.example/ws".to_string()),
1354 true,
1355 TransportBackend::Tungstenite,
1356 17,
1357 Some(ProxyUrl::parse(PROXY_URL).unwrap()),
1358 );
1359 let primary = pool.inner.market_client(true, PRIMARY_SHARD_ID);
1360 let secondary = pool.inner.market_client(false, PRIMARY_SHARD_ID + 1);
1361 let debug = format!("{pool:?}");
1362
1363 assert_eq!(pool.inner.proxy_url.as_ref().unwrap().expose(), PROXY_URL);
1364 assert_eq!(primary.proxy_url().unwrap().expose(), PROXY_URL);
1365 assert_eq!(secondary.proxy_url().unwrap().expose(), PROXY_URL);
1366 assert_eq!(pool.inner.max_subscriptions, 17);
1367 assert!(!debug.contains("pool-proxy-secret"));
1368 }
1369
1370 #[rstest]
1371 #[case::first_has_room(&[0, 200], 200, Some(0))]
1372 #[case::prefers_lowest_id(&[200, 5, 5], 200, Some(1))]
1373 #[case::all_full(&[200, 200], 200, None)]
1374 #[case::exact_boundary_is_full(&[1], 1, None)]
1375 fn smallest_shard_with_capacity_picks_lowest_open_id(
1376 #[case] owned: &[usize],
1377 #[case] max: usize,
1378 #[case] expected: Option<usize>,
1379 ) {
1380 let state = state_with_shards(owned);
1381 assert_eq!(smallest_shard_with_capacity(&state, max), expected);
1382 }
1383
1384 #[rstest]
1385 fn available_shard_id_reuses_lowest_closed_shard() {
1386 let mut state = state_with_shards(&[1, 1, 1]);
1387 state.shards.remove(&(PRIMARY_SHARD_ID + 1));
1388
1389 assert_eq!(available_shard_id(&state), PRIMARY_SHARD_ID + 1);
1390 }
1391
1392 #[rstest]
1393 #[tokio::test]
1394 async fn subscribe_routes_command_and_tracks_assignment() {
1395 let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
1396 let handle = Handle::test_single_shard(tx, &[]);
1397
1398 handle
1399 .subscribe_market(vec!["token-a".to_string()])
1400 .await
1401 .expect("subscribe");
1402
1403 match rx.try_recv().expect("expected SubscribeMarket") {
1404 HandlerCommand::SubscribeMarket(ids) => assert_eq!(ids, vec!["token-a".to_string()]),
1405 other => panic!("unexpected command: {other:?}"),
1406 }
1407 assert_eq!(handle.inner.subscription_count_for_test(), 1);
1408 }
1409
1410 #[rstest]
1411 #[tokio::test]
1412 async fn duplicate_subscribe_does_not_consume_capacity_or_resend() {
1413 let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
1414 let handle = Handle::test_single_shard(tx, &[]);
1415
1416 handle
1417 .subscribe_market(vec!["token-a".to_string()])
1418 .await
1419 .expect("first subscribe");
1420 handle
1421 .subscribe_market(vec!["token-a".to_string()])
1422 .await
1423 .expect("duplicate subscribe");
1424
1425 assert!(matches!(
1426 rx.try_recv(),
1427 Ok(HandlerCommand::SubscribeMarket(_))
1428 ));
1429 assert!(rx.try_recv().is_err(), "duplicate must not resend");
1430 assert_eq!(handle.inner.subscription_count_for_test(), 1);
1431 }
1432
1433 #[rstest]
1434 #[tokio::test]
1435 async fn unsubscribe_routes_command_and_releases_assignment() {
1436 let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
1437 let handle = Handle::test_single_shard(tx, &["token-a"]);
1438
1439 handle
1440 .unsubscribe_market(vec!["token-a".to_string()])
1441 .await
1442 .expect("unsubscribe");
1443
1444 match rx.try_recv().expect("expected UnsubscribeMarket") {
1445 HandlerCommand::UnsubscribeMarket(ids) => assert_eq!(ids, vec!["token-a".to_string()]),
1446 other => panic!("unexpected command: {other:?}"),
1447 }
1448 assert_eq!(handle.inner.subscription_count_for_test(), 0);
1449 }
1450
1451 #[rstest]
1452 #[tokio::test]
1453 async fn unsubscribe_unknown_token_is_noop() {
1454 let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
1455 let handle = Handle::test_single_shard(tx, &[]);
1456
1457 handle
1458 .unsubscribe_market(vec!["token-a".to_string()])
1459 .await
1460 .expect("unsubscribe");
1461
1462 assert!(rx.try_recv().is_err());
1463 }
1464
1465 #[rstest]
1466 #[tokio::test]
1467 async fn subscribe_send_failure_rolls_back_assignment() {
1468 let (tx, rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
1469 drop(rx);
1470 let handle = Handle::test_single_shard(tx, &[]);
1471
1472 let result = handle.subscribe_market(vec!["token-a".to_string()]).await;
1473
1474 assert!(result.is_err());
1475 assert_eq!(handle.inner.subscription_count_for_test(), 0);
1476 }
1477
1478 #[rstest]
1479 #[tokio::test]
1480 async fn resubscribe_market_cycles_without_changing_assignment() {
1481 let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
1482 let handle = Handle::test_single_shard(tx, &["token-a"]);
1483 let cancel = CancellationToken::new();
1484
1485 let task = {
1486 let handle = handle.clone();
1487 let cancel = cancel.clone();
1488
1489 tokio::spawn(async move {
1490 handle
1491 .resubscribe_market(
1492 vec!["token-a".to_string()],
1493 &cancel,
1494 &SnapshotGate::default(),
1495 )
1496 .await
1497 })
1498 };
1499
1500 match rx.recv().await.expect("expected cycle command") {
1501 HandlerCommand::CycleMarketSubscription {
1502 asset_ids,
1503 responder,
1504 ..
1505 } => {
1506 assert_eq!(asset_ids, vec!["token-a".to_string()]);
1507 responder
1508 .send(CycleMarketOutcome::Completed)
1509 .expect("answer cycle");
1510 }
1511 other => panic!("unexpected command: {other:?}"),
1512 }
1513
1514 assert!(matches!(
1515 task.await.expect("cycle task").expect("cycle owned asset"),
1516 CycleMarketOutcome::Completed
1517 ));
1518 assert_eq!(
1519 handle.tokens_for_shard(PRIMARY_SHARD_ID),
1520 vec![Ustr::from("token-a")]
1521 );
1522 assert!(handle.tokens_for_shard(PRIMARY_SHARD_ID + 1).is_empty());
1523 assert_eq!(handle.inner.subscription_count_for_test(), 1);
1524 }
1525
1526 #[rstest]
1527 #[tokio::test]
1528 async fn resubscribe_market_propagates_cycle_outcome() {
1529 let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
1530 let handle = Handle::test_single_shard(tx, &["token-a"]);
1531 let cancel = CancellationToken::new();
1532
1533 let task = {
1534 let handle = handle.clone();
1535 let cancel = cancel.clone();
1536
1537 tokio::spawn(async move {
1538 handle
1539 .resubscribe_market(
1540 vec!["token-a".to_string()],
1541 &cancel,
1542 &SnapshotGate::default(),
1543 )
1544 .await
1545 })
1546 };
1547
1548 match rx.recv().await.expect("expected cycle command") {
1549 HandlerCommand::CycleMarketSubscription { responder, .. } => {
1550 responder
1551 .send(CycleMarketOutcome::ConnectionChanged)
1552 .expect("answer cycle");
1553 }
1554 other => panic!("unexpected command: {other:?}"),
1555 }
1556
1557 assert!(matches!(
1558 task.await.expect("cycle task").expect("cycle reports"),
1559 CycleMarketOutcome::ConnectionChanged
1560 ));
1561 assert_eq!(
1563 handle.tokens_for_shard(PRIMARY_SHARD_ID),
1564 vec![Ustr::from("token-a")]
1565 );
1566 }
1567
1568 #[rstest]
1569 #[tokio::test]
1570 async fn resubscribe_market_fails_when_cycle_response_lost() {
1571 let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
1572 let handle = Handle::test_single_shard(tx, &["token-a"]);
1573 let cancel = CancellationToken::new();
1574
1575 let task = {
1576 let handle = handle.clone();
1577 let cancel = cancel.clone();
1578
1579 tokio::spawn(async move {
1580 handle
1581 .resubscribe_market(
1582 vec!["token-a".to_string()],
1583 &cancel,
1584 &SnapshotGate::default(),
1585 )
1586 .await
1587 })
1588 };
1589
1590 assert!(matches!(
1592 rx.recv().await.expect("expected cycle command"),
1593 HandlerCommand::CycleMarketSubscription { .. }
1594 ));
1595
1596 let err = task.await.expect("cycle task").expect_err("lost response");
1597
1598 assert!(
1599 err.to_string().contains("response lost"),
1600 "unexpected error: {err}"
1601 );
1602 }
1603
1604 #[rstest]
1605 #[tokio::test]
1606 async fn resubscribe_market_rejects_unowned_asset() {
1607 let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
1608 let handle = Handle::test_single_shard(tx, &["token-a"]);
1609
1610 let err = handle
1611 .resubscribe_market(
1612 vec!["token-unknown".to_string()],
1613 &CancellationToken::new(),
1614 &SnapshotGate::default(),
1615 )
1616 .await
1617 .expect_err("unowned asset must fail");
1618
1619 assert!(
1620 err.to_string().contains("not owned by any shard"),
1621 "unexpected error: {err}"
1622 );
1623 assert!(rx.try_recv().is_err());
1624 }
1625
1626 #[rstest]
1627 #[tokio::test]
1628 async fn resubscribe_market_rejects_missing_shard() {
1629 let (tx, _rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
1630 let handle = Handle::test_single_shard(tx, &["token-a"]);
1631 handle.inner.state.lock().shards.remove(&PRIMARY_SHARD_ID);
1632
1633 let err = handle
1634 .resubscribe_market(
1635 vec!["token-a".to_string()],
1636 &CancellationToken::new(),
1637 &SnapshotGate::default(),
1638 )
1639 .await
1640 .expect_err("missing shard must fail");
1641
1642 assert!(
1643 err.to_string().contains("is not available"),
1644 "unexpected error: {err}"
1645 );
1646 }
1647
1648 #[rstest]
1649 #[tokio::test]
1650 async fn resubscribe_market_rejects_closed_pool() {
1651 let (tx, _rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
1652 let handle = Handle::test_single_shard(tx, &["token-a"]);
1653 handle.inner.closed.store(true, Ordering::SeqCst);
1654
1655 let err = handle
1656 .resubscribe_market(
1657 vec!["token-a".to_string()],
1658 &CancellationToken::new(),
1659 &SnapshotGate::default(),
1660 )
1661 .await
1662 .expect_err("closed pool must fail");
1663
1664 assert!(
1665 err.to_string().contains("closed"),
1666 "unexpected error: {err}"
1667 );
1668 }
1669
1670 #[rstest]
1671 #[tokio::test]
1672 async fn forwarder_stamps_reconnected_with_shard_id() {
1673 let inner = PoolInner::new(None, TransportBackend::default(), true, 1);
1674 let (out_tx, mut out_rx) = tokio::sync::mpsc::unbounded_channel();
1675 *inner.out_tx.lock() = Some(out_tx);
1676 let (shard_tx, shard_rx) = tokio::sync::mpsc::unbounded_channel();
1677 let mut forwarder = inner
1678 .spawn_forwarder(shard_rx, PRIMARY_SHARD_ID + 1)
1679 .expect("spawn forwarder");
1680
1681 shard_tx
1682 .send(PolymarketWsMessage::Reconnected { shard_id: None })
1683 .unwrap();
1684 shard_tx
1685 .send(market_message("ws_market_best_bid_ask_msg.json"))
1686 .unwrap();
1687 drop(shard_tx);
1688 let outcome = finish_task(
1689 &mut forwarder,
1690 Duration::from_secs(1),
1691 Duration::from_secs(1),
1692 )
1693 .await
1694 .expect("forwarder task");
1695 assert!(matches!(outcome, TaskJoinOutcome::Completed(())));
1696
1697 assert!(
1698 matches!(
1699 out_rx.try_recv().unwrap(),
1700 PolymarketWsMessage::Reconnected { shard_id: Some(id) }
1701 if id == PRIMARY_SHARD_ID + 1
1702 ),
1703 "forwarder must stamp the reconnecting shard"
1704 );
1705 assert!(matches!(
1706 out_rx.try_recv().unwrap(),
1707 PolymarketWsMessage::Market(MarketWsMessage::BestBidAsk(_))
1708 ));
1709 assert!(out_rx.try_recv().is_err());
1710 }
1711}