Skip to main content

nautilus_polymarket/websocket/
pool.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16//! Market-channel WebSocket connection pool for the Polymarket CLOB API.
17//!
18//! [`PolymarketWebSocketClient`] stays a single-channel, single-connection
19//! primitive. This pool owns a set of market-channel connections (shards) and
20//! spreads unique asset subscriptions across them so no single connection carries
21//! more than `ws_max_subscriptions` assets. See [`WS_DEFAULT_SUBSCRIPTIONS`] for
22//! why that bound exists.
23//!
24//! The pool grows lazily: it starts with one shard and opens another only when the
25//! current shards are full at subscribe time. A secondary shard closes once it owns
26//! no assets; the primary shard (which carries new-market discovery) always
27//! persists. Each shard replays only its own subscriptions on reconnect because
28//! that state lives inside its own [`PolymarketWebSocketClient`]. When custom
29//! features are enabled, every shard requests asset-scoped best-bid/ask events,
30//! while secondary shards discard global discovery and resolution events.
31
32use std::sync::{
33    Arc,
34    atomic::{AtomicBool, Ordering},
35};
36
37use ahash::AHashMap;
38use futures_util::{StreamExt, stream::FuturesUnordered};
39use nautilus_live::{
40    SocketControlFactory,
41    book::snapshot::SnapshotGate,
42    task::{TaskJoinOutcome, TaskSlot, TaskSpawnError, finish_task},
43};
44use nautilus_network::websocket::{TransportBackend, proxy::ProxyUrl};
45use parking_lot::Mutex;
46use ustr::Ustr;
47
48use super::{
49    MARKET_STREAMS_ENDPOINT,
50    client::{PolymarketWebSocketClient, WsSubscriptionHandle},
51    handler::CycleMarketOutcome,
52    messages::{MarketWsMessage, PolymarketWsMessage},
53};
54use crate::common::consts::WS_DEFAULT_SUBSCRIPTIONS;
55
56// Primary shard carries new-market discovery and never auto-closes.
57const PRIMARY_SHARD_ID: usize = 0;
58
59/// A pool of market-channel WebSocket connections that shards asset subscriptions.
60#[derive(Debug)]
61pub struct PolymarketMarketConnectionPool {
62    inner: Arc<PoolInner>,
63}
64
65/// Cloneable routing handle used from spawned subscription tasks.
66///
67/// Routes each asset to its owning shard and grows the pool on demand.
68#[derive(Clone, Debug)]
69pub struct PolymarketMarketPoolHandle {
70    inner: Arc<PoolInner>,
71}
72
73#[derive(Debug)]
74struct PoolInner {
75    base_url: Option<String>,
76    proxy_url: Option<ProxyUrl>,
77    transport_backend: TransportBackend,
78    subscribe_new_markets: bool,
79    max_subscriptions: usize,
80    connect_timeout_secs: Mutex<Option<u64>>,
81    // Serializes routing and shard growth; held across the async wire sends.
82    wire_mutex: tokio::sync::Mutex<()>,
83    // Never locked across an await, so routing futures stay `Send`.
84    state: Mutex<PoolState>,
85    out_tx: Mutex<Option<tokio::sync::mpsc::UnboundedSender<PolymarketWsMessage>>>,
86    out_rx: Mutex<Option<tokio::sync::mpsc::UnboundedReceiver<PolymarketWsMessage>>>,
87    socket_factory: Mutex<Option<SocketControlFactory>>,
88    closed: AtomicBool,
89}
90
91#[derive(Debug, Default)]
92struct PoolState {
93    shards: AHashMap<usize, ShardEntry>,
94    assignments: AHashMap<Ustr, usize>,
95    shutdown_errors: Vec<String>,
96}
97
98struct PoolDrain<'a> {
99    owner: &'a Mutex<PoolState>,
100    state: PoolState,
101    completed: Vec<usize>,
102}
103
104impl<'a> PoolDrain<'a> {
105    fn take(owner: &'a Mutex<PoolState>) -> Self {
106        let state = std::mem::take(&mut *owner.lock());
107        Self {
108            owner,
109            state,
110            completed: Vec::new(),
111        }
112    }
113}
114
115impl Drop for PoolDrain<'_> {
116    fn drop(&mut self) {
117        for id in &self.completed {
118            self.state.shards.remove(id);
119        }
120
121        // Prune once, including when cancellation interrupts the drain
122        self.state
123            .assignments
124            .retain(|_, id| self.state.shards.contains_key(id));
125        *self.owner.lock() = std::mem::take(&mut self.state);
126    }
127}
128
129#[derive(Debug)]
130struct ShardEntry {
131    client: PolymarketWebSocketClient,
132    handle: WsSubscriptionHandle,
133    forwarder: TaskSlot<()>,
134    owned: usize,
135    closing: bool,
136}
137
138enum ReleaseOutcome {
139    NotOwned,
140    Unsubscribe(WsSubscriptionHandle),
141    CloseShard(usize, Box<ShardEntry>),
142}
143
144struct ShardClose<'a> {
145    owner: &'a Mutex<PoolState>,
146    id: usize,
147    shard: Option<Box<ShardEntry>>,
148}
149
150impl<'a> ShardClose<'a> {
151    fn new(owner: &'a Mutex<PoolState>, id: usize, mut shard: Box<ShardEntry>) -> Self {
152        shard.closing = true;
153        Self {
154            owner,
155            id,
156            shard: Some(shard),
157        }
158    }
159
160    fn shard_mut(&mut self) -> &mut ShardEntry {
161        self.shard.as_deref_mut().expect("closing shard present")
162    }
163
164    fn complete(mut self) {
165        self.shard.take();
166    }
167}
168
169impl Drop for ShardClose<'_> {
170    fn drop(&mut self) {
171        if let Some(shard) = self.shard.take() {
172            let replaced = self.owner.lock().shards.insert(self.id, *shard);
173            assert!(replaced.is_none(), "closing shard ID is already present");
174        }
175    }
176}
177
178#[allow(
179    clippy::missing_panics_doc,
180    reason = "internal mutex locks and shard-state invariants are not expected to panic"
181)]
182impl PolymarketMarketConnectionPool {
183    /// Creates a new market connection pool (unconnected).
184    ///
185    /// A `max_subscriptions` of `0` is invalid and clamps to
186    /// [`WS_DEFAULT_SUBSCRIPTIONS`] with a warning.
187    #[must_use]
188    pub fn new(
189        base_url: Option<String>,
190        subscribe_new_markets: bool,
191        transport_backend: TransportBackend,
192        max_subscriptions: usize,
193    ) -> Self {
194        Self::new_with_proxy(
195            base_url,
196            subscribe_new_markets,
197            transport_backend,
198            max_subscriptions,
199            None,
200        )
201    }
202
203    /// Creates a new market connection pool with an optional validated proxy URL.
204    #[must_use]
205    pub fn new_with_proxy(
206        base_url: Option<String>,
207        subscribe_new_markets: bool,
208        transport_backend: TransportBackend,
209        max_subscriptions: usize,
210        proxy_url: Option<ProxyUrl>,
211    ) -> Self {
212        Self {
213            inner: Arc::new(PoolInner::new_with_proxy(
214                base_url,
215                transport_backend,
216                subscribe_new_markets,
217                max_subscriptions,
218                proxy_url,
219            )),
220        }
221    }
222
223    pub(crate) fn with_connect_timeout(self, timeout_secs: u64) -> Self {
224        *self.inner.connect_timeout_secs.lock() = Some(timeout_secs);
225        self
226    }
227
228    /// Configures socket state reporting and reconnect control for every connection in the pool.
229    #[must_use]
230    pub(crate) fn with_socket_factory(self, factory: SocketControlFactory) -> Self {
231        *self.inner.socket_factory.lock() = Some(factory);
232        self
233    }
234
235    #[cfg(test)]
236    pub(crate) fn proxy_url(&self) -> Option<&ProxyUrl> {
237        self.inner.proxy_url.as_ref()
238    }
239
240    /// Returns a cloneable routing handle for use in spawned subscription tasks.
241    #[must_use]
242    pub fn handle(&self) -> PolymarketMarketPoolHandle {
243        PolymarketMarketPoolHandle {
244            inner: Arc::clone(&self.inner),
245        }
246    }
247
248    pub(crate) async fn prepare(&self) -> anyhow::Result<()> {
249        if self.inner.closed.load(Ordering::Acquire) {
250            self.disconnect().await?;
251        }
252
253        let _wire = self.inner.wire_mutex.lock().await;
254        self.inner.prepare_channel();
255        Ok(())
256    }
257
258    /// Opens the primary shard and prepares the merged message stream.
259    ///
260    /// # Errors
261    ///
262    /// Returns an error if the primary connection cannot be established.
263    pub async fn connect(&self) -> anyhow::Result<()> {
264        if self.inner.closed.load(Ordering::Acquire) {
265            self.disconnect().await?;
266        }
267
268        let _wire = self.inner.wire_mutex.lock().await;
269
270        if !self.inner.closed.load(Ordering::Acquire) && !self.inner.state.lock().shards.is_empty()
271        {
272            log::warn!("Polymarket market pool already connected");
273            return Ok(());
274        }
275
276        self.inner.prepare_channel();
277        self.inner.connect_new_shard(true).await?;
278        Ok(())
279    }
280
281    /// Sends the new-market discovery subscribe on the primary shard.
282    ///
283    /// # Errors
284    ///
285    /// Returns an error if no primary shard is available.
286    pub async fn subscribe_new_markets_feed(&self) -> anyhow::Result<()> {
287        let _wire = self.inner.wire_mutex.lock().await;
288
289        let handle = {
290            let state = self.inner.state.lock();
291            if self.inner.closed.load(Ordering::Acquire) {
292                anyhow::bail!("Market connection pool is closed");
293            }
294            state
295                .shards
296                .get(&PRIMARY_SHARD_ID)
297                .map(|shard| shard.handle.clone())
298        };
299
300        match handle {
301            Some(handle) => handle.subscribe_market(vec![]).await,
302            None => anyhow::bail!("No primary market shard available for new-market discovery"),
303        }
304    }
305
306    /// Takes the merged message receiver, leaving `None` in its place.
307    #[must_use]
308    pub fn take_message_receiver(
309        &self,
310    ) -> Option<tokio::sync::mpsc::UnboundedReceiver<PolymarketWsMessage>> {
311        self.inner.out_rx.lock().take()
312    }
313
314    /// Disconnects every shard and clears routing state.
315    ///
316    /// # Errors
317    ///
318    /// Returns an error if a task or connection does not stop. Attempts every shard before
319    /// returning the error.
320    pub async fn disconnect(&self) -> anyhow::Result<()> {
321        self.inner.begin_shutdown();
322        let _wire = self.inner.wire_mutex.lock().await;
323
324        let mut drain = PoolDrain::take(&self.inner.state);
325        {
326            let errors = Mutex::new(&mut drain.state.shutdown_errors);
327            let mut disconnects = drain
328                .state
329                .shards
330                .iter_mut()
331                .map(|(&shard_id, shard)| {
332                    let errors = &errors;
333                    async move {
334                        let mut failed = false;
335                        shard.forwarder.abort();
336                        if let Some(outcome) = finish_task(
337                            &mut shard.forwarder,
338                            std::time::Duration::ZERO,
339                            std::time::Duration::from_secs(2),
340                        )
341                        .await
342                        {
343                            match outcome {
344                                TaskJoinOutcome::Completed(()) | TaskJoinOutcome::Aborted => {}
345                                TaskJoinOutcome::Failed(error) => {
346                                    failed = true;
347                                    errors.lock().push(format!(
348                                        "market shard {shard_id} forwarder failed: {error}"
349                                    ));
350                                }
351                                TaskJoinOutcome::Incomplete => {
352                                    failed = true;
353                                    errors.lock().push(format!(
354                                        "market shard {shard_id} forwarder did not stop after abort"
355                                    ));
356                                }
357                            }
358                        }
359
360                        if let Err(e) = shard.client.disconnect().await {
361                            failed = true;
362                            errors
363                                .lock()
364                                .push(format!("market shard {shard_id} disconnect failed: {e}"));
365                        }
366                        (shard_id, failed)
367                    }
368                })
369                .collect::<FuturesUnordered<_>>();
370
371            while let Some((shard_id, failed)) = disconnects.next().await {
372                if !failed {
373                    drain.completed.push(shard_id);
374                }
375            }
376        }
377
378        if !drain.state.shutdown_errors.is_empty() {
379            let errors = std::mem::take(&mut drain.state.shutdown_errors);
380            anyhow::bail!(
381                "Polymarket market pool shutdown failed: {}",
382                errors.join("; ")
383            );
384        }
385
386        *self.inner.out_tx.lock() = None;
387        *self.inner.out_rx.lock() = None;
388        Ok(())
389    }
390
391    pub(crate) fn begin_shutdown(&self) {
392        self.inner.begin_shutdown();
393    }
394
395    /// Clears retained reconnect-replay state on any remaining shards.
396    pub(crate) fn clear_reconnect_state(&self) {
397        let state = self.inner.state.lock();
398        for shard in state.shards.values() {
399            shard.client.clear_reconnect_state();
400        }
401    }
402
403    /// Returns the number of open shard connections.
404    #[must_use]
405    pub fn connection_count(&self) -> usize {
406        self.inner.state.lock().shards.len()
407    }
408
409    /// Returns the number of unique assets assigned across all shards.
410    #[must_use]
411    pub fn subscription_count(&self) -> usize {
412        self.inner.state.lock().assignments.len()
413    }
414}
415
416impl Drop for PoolInner {
417    fn drop(&mut self) {
418        self.closed.store(true, Ordering::Release);
419
420        for shard in self.state.get_mut().shards.values_mut() {
421            shard.forwarder.abort();
422            shard.client.abort();
423        }
424    }
425}
426
427#[allow(
428    clippy::missing_panics_doc,
429    reason = "internal mutex locks and shard-state invariants are not expected to panic"
430)]
431impl PolymarketMarketPoolHandle {
432    pub(crate) fn begin_shutdown(&self) {
433        self.inner.begin_shutdown();
434    }
435
436    /// Subscribes to market data for the given asset IDs, sharding across connections.
437    ///
438    /// # Errors
439    ///
440    /// Returns an error if a shard cannot be opened or a subscribe send fails.
441    pub async fn subscribe_market(&self, asset_ids: Vec<String>) -> anyhow::Result<()> {
442        let _wire = self.inner.wire_mutex.lock().await;
443        self.inner.ensure_open()?;
444        for asset_id in asset_ids {
445            self.inner.subscribe_one(asset_id).await?;
446        }
447        Ok(())
448    }
449
450    /// Removes asset IDs from their owning shards, closing emptied secondary shards.
451    ///
452    /// # Errors
453    ///
454    /// Returns an error if an unsubscribe send fails.
455    pub async fn unsubscribe_market(&self, asset_ids: Vec<String>) -> anyhow::Result<()> {
456        let _wire = self.inner.wire_mutex.lock().await;
457        self.inner.ensure_open()?;
458        for asset_id in asset_ids {
459            self.inner.unsubscribe_one(asset_id).await?;
460        }
461        Ok(())
462    }
463
464    /// Cycles owned assets (unsubscribe then subscribe) to trigger fresh snapshots.
465    ///
466    /// Unlike [`Self::subscribe_market`], this never changes shard assignment: it routes
467    /// each asset to its owning shard, which re-subscribes it on the wire without
468    /// touching desired subscription state. The venue ignores a duplicate subscribe,
469    /// so only a real cycle produces a fresh `book` snapshot. Used by book recovery.
470    ///
471    /// # Errors
472    ///
473    /// Returns an error if the pool is closed, an asset is not owned by any shard,
474    /// or a cycle command send fails.
475    pub async fn resubscribe_market(
476        &self,
477        asset_ids: Vec<String>,
478        cancel: &tokio_util::sync::CancellationToken,
479        gate: &SnapshotGate,
480    ) -> anyhow::Result<CycleMarketOutcome> {
481        let mut outcome = CycleMarketOutcome::Completed;
482
483        for asset_id in asset_ids {
484            outcome = self.inner.cycle_one(&asset_id, cancel, gate).await?;
485
486            if !matches!(outcome, CycleMarketOutcome::Completed) {
487                break;
488            }
489        }
490
491        Ok(outcome)
492    }
493
494    /// Returns the tokens currently assigned to `shard_id`.
495    pub(crate) fn tokens_for_shard(&self, shard_id: usize) -> Vec<Ustr> {
496        let state = self.inner.state.lock();
497        state
498            .assignments
499            .iter()
500            .filter_map(|(token, assigned)| (*assigned == shard_id).then_some(*token))
501            .collect()
502    }
503}
504
505impl PoolInner {
506    fn begin_shutdown(&self) {
507        let state = self.state.lock();
508        self.closed.store(true, Ordering::Release);
509
510        for shard in state.shards.values() {
511            shard.client.begin_shutdown();
512        }
513    }
514
515    fn prepare_channel(&self) {
516        {
517            let _state = self.state.lock();
518            self.closed.store(false, Ordering::Release);
519        }
520
521        if self.out_tx.lock().is_some() {
522            return;
523        }
524
525        let (out_tx, out_rx) = tokio::sync::mpsc::unbounded_channel();
526        *self.out_tx.lock() = Some(out_tx);
527        *self.out_rx.lock() = Some(out_rx);
528    }
529
530    fn ensure_open(&self) -> anyhow::Result<()> {
531        let _state = self.state.lock();
532
533        if self.closed.load(Ordering::Acquire) {
534            anyhow::bail!("Market connection pool is closed");
535        }
536        Ok(())
537    }
538}
539
540impl PoolInner {
541    #[cfg(test)]
542    fn new(
543        base_url: Option<String>,
544        transport_backend: TransportBackend,
545        subscribe_new_markets: bool,
546        max_subscriptions: usize,
547    ) -> Self {
548        Self::new_with_proxy(
549            base_url,
550            transport_backend,
551            subscribe_new_markets,
552            max_subscriptions,
553            None,
554        )
555    }
556
557    fn new_with_proxy(
558        base_url: Option<String>,
559        transport_backend: TransportBackend,
560        subscribe_new_markets: bool,
561        max_subscriptions: usize,
562        proxy_url: Option<ProxyUrl>,
563    ) -> Self {
564        let max_subscriptions = if max_subscriptions == 0 {
565            log::warn!(
566                "PolymarketDataClientConfig.ws_max_subscriptions=0 is invalid, using {WS_DEFAULT_SUBSCRIPTIONS}"
567            );
568            WS_DEFAULT_SUBSCRIPTIONS
569        } else {
570            max_subscriptions
571        };
572
573        Self {
574            base_url,
575            proxy_url,
576            transport_backend,
577            subscribe_new_markets,
578            max_subscriptions,
579            connect_timeout_secs: Mutex::new(None),
580            wire_mutex: tokio::sync::Mutex::new(()),
581            state: Mutex::new(PoolState::default()),
582            out_tx: Mutex::new(None),
583            out_rx: Mutex::new(None),
584            socket_factory: Mutex::new(None),
585            closed: AtomicBool::new(false),
586        }
587    }
588
589    // Callers hold `wire_mutex`.
590    async fn subscribe_one(&self, asset_id: String) -> anyhow::Result<()> {
591        let token = Ustr::from(asset_id.as_str());
592
593        let Some(handle) = self.assign(token).await? else {
594            return Ok(());
595        };
596
597        let result = match self.ensure_open() {
598            Ok(()) => handle.subscribe_market(vec![asset_id]).await,
599            Err(e) => Err(e),
600        };
601
602        if let Err(e) = result {
603            // Roll back so a failed send leaves no stale assignment or empty shard.
604            if let ReleaseOutcome::CloseShard(id, shard) = self.release(token)
605                && let Err(close_error) = self.close_shard(id, shard).await
606            {
607                anyhow::bail!("{e}; subscription rollback failed: {close_error}");
608            }
609            return Err(e);
610        }
611        Ok(())
612    }
613
614    // Callers hold `wire_mutex`.
615    async fn unsubscribe_one(&self, asset_id: String) -> anyhow::Result<()> {
616        self.ensure_open()?;
617        let token = Ustr::from(asset_id.as_str());
618
619        match self.release(token) {
620            ReleaseOutcome::NotOwned => Ok(()),
621            ReleaseOutcome::Unsubscribe(handle) => handle.unsubscribe_market(vec![asset_id]).await,
622            ReleaseOutcome::CloseShard(id, shard) => {
623                // Disconnect drops the shard's subscriptions; no unsubscribe send needed.
624                self.close_shard(id, shard).await
625            }
626        }
627    }
628
629    async fn cycle_one(
630        &self,
631        asset_id: &str,
632        cancel: &tokio_util::sync::CancellationToken,
633        gate: &SnapshotGate,
634    ) -> anyhow::Result<CycleMarketOutcome> {
635        let responder_rx = {
636            let _wire = self.wire_mutex.lock().await;
637            self.ensure_open()?;
638
639            let handle = {
640                let state = self.state.lock();
641                let token = Ustr::from(asset_id);
642
643                let Some(id) = state.assignments.get(&token).copied() else {
644                    anyhow::bail!("Market asset {asset_id} is not owned by any shard");
645                };
646
647                state
648                    .shards
649                    .get(&id)
650                    .map(|shard| shard.handle.clone())
651                    .ok_or_else(|| anyhow::anyhow!("Market shard {id} is not available"))?
652            };
653
654            // The wire mutex is released before awaiting completion: the queued
655            // command is ordered, and awaiting must not block pool operations.
656            let (responder_tx, responder_rx) = tokio::sync::oneshot::channel();
657            handle
658                .cycle_market_subscription(
659                    vec![asset_id.to_string()],
660                    cancel.clone(),
661                    responder_tx,
662                    gate.clone(),
663                )
664                .await?;
665            responder_rx
666        };
667
668        responder_rx
669            .await
670            .map_err(|_| anyhow::anyhow!("Market subscription cycle response lost"))
671    }
672
673    // Returns `None` when the token is already owned by a shard.
674    async fn assign(&self, token: Ustr) -> anyhow::Result<Option<WsSubscriptionHandle>> {
675        {
676            let mut state = self.state.lock();
677
678            if self.closed.load(Ordering::Acquire) {
679                anyhow::bail!("Market connection pool is closed");
680            }
681
682            if state.assignments.contains_key(&token) {
683                return Ok(None);
684            }
685
686            if let Some(id) = smallest_shard_with_capacity(&state, self.max_subscriptions) {
687                let handle = {
688                    let shard = state.shards.get_mut(&id).expect("shard present");
689                    shard.owned += 1;
690                    shard.handle.clone()
691                };
692                state.assignments.insert(token, id);
693                return Ok(Some(handle));
694            }
695        }
696
697        let id = self.connect_new_shard(false).await?;
698
699        let rejected_shard = {
700            let mut state = self.state.lock();
701
702            if !self.closed.load(Ordering::Acquire) {
703                let handle = {
704                    let shard = state.shards.get_mut(&id).expect("new shard present");
705                    shard.owned += 1;
706                    shard.handle.clone()
707                };
708                state.assignments.insert(token, id);
709                return Ok(Some(handle));
710            }
711
712            state
713                .shards
714                .remove(&id)
715                .expect("new shard retained for shutdown")
716        };
717
718        if let Err(e) = self.close_shard(id, Box::new(rejected_shard)).await {
719            anyhow::bail!("Market connection pool is closed; shard rollback failed: {e}");
720        }
721
722        anyhow::bail!("Market connection pool is closed")
723    }
724
725    fn release(&self, token: Ustr) -> ReleaseOutcome {
726        let mut state = self.state.lock();
727
728        let Some(id) = state.assignments.remove(&token) else {
729            return ReleaseOutcome::NotOwned;
730        };
731
732        let owned = {
733            let Some(shard) = state.shards.get_mut(&id) else {
734                return ReleaseOutcome::NotOwned;
735            };
736            shard.owned = shard.owned.saturating_sub(1);
737            shard.owned
738        };
739
740        if id != PRIMARY_SHARD_ID && owned == 0 {
741            let shard = state.shards.remove(&id).expect("shard present");
742            ReleaseOutcome::CloseShard(id, Box::new(shard))
743        } else {
744            let handle = state.shards.get(&id).expect("shard present").handle.clone();
745            ReleaseOutcome::Unsubscribe(handle)
746        }
747    }
748
749    async fn connect_new_shard(&self, is_primary: bool) -> anyhow::Result<usize> {
750        if self.closed.load(Ordering::Acquire) {
751            anyhow::bail!("Market connection pool is closed");
752        }
753
754        let id = if is_primary {
755            PRIMARY_SHARD_ID
756        } else {
757            let state = self.state.lock();
758            available_shard_id(&state)
759        };
760
761        let mut client = self.market_client(self.subscribe_new_markets, id);
762        client.connect().await?;
763
764        let handle = client.clone_subscription_handle();
765        let rx = client
766            .take_message_receiver()
767            .ok_or_else(|| anyhow::anyhow!("Market shard receiver unavailable after connect"))?;
768
769        let forwarder = match self.spawn_forwarder(rx, id) {
770            Ok(forwarder) => forwarder,
771            Err((e, forwarder)) => {
772                let shard = Box::new(ShardEntry {
773                    client,
774                    handle,
775                    forwarder,
776                    owned: 0,
777                    closing: true,
778                });
779
780                if let Err(close_error) = self.close_shard(id, shard).await {
781                    anyhow::bail!(
782                        "Failed to start market shard forwarder: {e}; startup rollback failed: \
783                         {close_error}"
784                    );
785                }
786                anyhow::bail!("Failed to start market shard forwarder: {e}");
787            }
788        };
789
790        let shard = ShardEntry {
791            client,
792            handle,
793            forwarder,
794            owned: 0,
795            closing: false,
796        };
797        let rejected_shard = {
798            let mut state = self.state.lock();
799
800            if self.closed.load(Ordering::Acquire) {
801                Some(shard)
802            } else {
803                state.shards.insert(id, shard);
804                None
805            }
806        };
807
808        if let Some(shard) = rejected_shard {
809            if let Err(e) = self.close_shard(id, Box::new(shard)).await {
810                anyhow::bail!("Market connection pool is closed; shard rollback failed: {e}");
811            }
812            anyhow::bail!("Market connection pool is closed");
813        }
814
815        log::debug!("Opened Polymarket market shard {id}");
816        Ok(id)
817    }
818
819    fn market_client(
820        &self,
821        subscribe_new_markets: bool,
822        shard_id: usize,
823    ) -> PolymarketWebSocketClient {
824        let client = PolymarketWebSocketClient::new_market_with_proxy(
825            self.base_url.clone(),
826            subscribe_new_markets,
827            self.transport_backend,
828            self.proxy_url.clone(),
829        );
830
831        let client = match *self.connect_timeout_secs.lock() {
832            Some(timeout_secs) => client.with_connect_timeout(timeout_secs),
833            None => client,
834        };
835
836        let factory = self.socket_factory.lock().clone();
837
838        if let Some(factory) = factory {
839            let endpoint = if shard_id == PRIMARY_SHARD_ID {
840                MARKET_STREAMS_ENDPOINT.to_string()
841            } else {
842                format!("{MARKET_STREAMS_ENDPOINT}-{shard_id}")
843            };
844            client.with_socket_control(factory.control(endpoint))
845        } else {
846            client
847        }
848    }
849
850    fn spawn_forwarder(
851        &self,
852        mut rx: tokio::sync::mpsc::UnboundedReceiver<PolymarketWsMessage>,
853        shard_id: usize,
854    ) -> Result<TaskSlot<()>, (TaskSpawnError, TaskSlot<()>)> {
855        let out_tx = self.out_tx.lock().clone();
856        let is_primary = shard_id == PRIMARY_SHARD_ID;
857
858        let mut forwarder = TaskSlot::new();
859        if let Err(e) = forwarder.spawn(async move {
860            let Some(out_tx) = out_tx else {
861                return;
862            };
863
864            while let Some(msg) = rx.recv().await {
865                if !should_forward_from_shard(&msg, is_primary) {
866                    continue;
867                }
868
869                let msg = match msg {
870                    PolymarketWsMessage::Reconnected { .. } => PolymarketWsMessage::Reconnected {
871                        shard_id: Some(shard_id),
872                    },
873                    other => other,
874                };
875
876                if out_tx.send(msg).is_err() {
877                    break;
878                }
879            }
880        }) {
881            return Err((e, forwarder));
882        }
883        Ok(forwarder)
884    }
885
886    async fn close_shard(&self, id: usize, shard: Box<ShardEntry>) -> anyhow::Result<()> {
887        let mut close = ShardClose::new(&self.state, id, shard);
888        close.shard_mut().forwarder.abort();
889        let forwarder_stopped = match finish_task(
890            &mut close.shard_mut().forwarder,
891            std::time::Duration::ZERO,
892            std::time::Duration::from_secs(2),
893        )
894        .await
895        {
896            None | Some(TaskJoinOutcome::Completed(()) | TaskJoinOutcome::Aborted) => true,
897            Some(TaskJoinOutcome::Failed(error)) => {
898                let error = format!("market shard {id} forwarder failed: {error}");
899                self.state.lock().shutdown_errors.push(error);
900                true
901            }
902            Some(TaskJoinOutcome::Incomplete) => {
903                let error = format!("market shard {id} forwarder did not stop after abort");
904                self.state.lock().shutdown_errors.push(error);
905                false
906            }
907        };
908
909        if let Err(e) = close.shard_mut().client.disconnect().await {
910            let error = format!("market shard {id} disconnect failed: {e}");
911            self.state.lock().shutdown_errors.push(error);
912        }
913
914        if forwarder_stopped && !close.shard_mut().client.has_task() {
915            close.complete();
916        }
917
918        let errors = std::mem::take(&mut self.state.lock().shutdown_errors);
919
920        if errors.is_empty() {
921            Ok(())
922        } else {
923            anyhow::bail!(errors.join("; "))
924        }
925    }
926
927    #[cfg(test)]
928    fn subscription_count_for_test(&self) -> usize {
929        self.state.lock().assignments.len()
930    }
931}
932
933fn should_forward_from_shard(message: &PolymarketWsMessage, is_primary: bool) -> bool {
934    is_primary
935        || !matches!(
936            message,
937            PolymarketWsMessage::Market(
938                MarketWsMessage::NewMarket(_) | MarketWsMessage::MarketResolved(_)
939            )
940        )
941}
942
943fn smallest_shard_with_capacity(state: &PoolState, max_subscriptions: usize) -> Option<usize> {
944    state
945        .shards
946        .iter()
947        .filter(|(_, shard)| !shard.closing && shard.owned < max_subscriptions)
948        .map(|(id, _)| *id)
949        .min()
950}
951
952fn available_shard_id(state: &PoolState) -> usize {
953    let mut id = PRIMARY_SHARD_ID + 1;
954    while state.shards.contains_key(&id) {
955        id = id.checked_add(1).expect("market shard ID space exhausted");
956    }
957    id
958}
959
960#[cfg(test)]
961impl PolymarketMarketPoolHandle {
962    /// In-memory single-shard handle backed by `sender`, `assigned` tokens
963    /// pre-owned. Never connected, so growth is never triggered.
964    pub(crate) fn test_single_shard(
965        sender: tokio::sync::mpsc::UnboundedSender<super::handler::HandlerCommand>,
966        assigned: &[&str],
967    ) -> Self {
968        let inner = PoolInner::new(
969            None,
970            TransportBackend::default(),
971            false,
972            WS_DEFAULT_SUBSCRIPTIONS,
973        );
974        {
975            let mut state = inner.state.lock();
976            state.shards.insert(
977                PRIMARY_SHARD_ID,
978                ShardEntry {
979                    client: PolymarketWebSocketClient::new_market(
980                        None,
981                        false,
982                        TransportBackend::default(),
983                    ),
984                    handle: WsSubscriptionHandle::from_sender(sender),
985                    forwarder: TaskSlot::new(),
986                    owned: assigned.len(),
987                    closing: false,
988                },
989            );
990
991            for token in assigned {
992                state
993                    .assignments
994                    .insert(Ustr::from(token), PRIMARY_SHARD_ID);
995            }
996        }
997        Self {
998            inner: Arc::new(inner),
999        }
1000    }
1001}
1002
1003#[cfg(test)]
1004mod tests {
1005    use std::{sync::Arc as StdArc, time::Duration};
1006
1007    use PolymarketMarketPoolHandle as Handle;
1008    use nautilus_network::transport::TransportError;
1009    use parking_lot::{Condvar, Mutex as TestMutex};
1010    use rstest::rstest;
1011    use tokio_util::sync::CancellationToken;
1012
1013    use super::*;
1014    use crate::websocket::handler::HandlerCommand;
1015
1016    struct BlockingDrop(StdArc<(TestMutex<(bool, bool)>, Condvar)>);
1017
1018    impl Drop for BlockingDrop {
1019        fn drop(&mut self) {
1020            let (state, wake) = &*self.0;
1021            let mut state = state.lock();
1022            state.0 = true;
1023            wake.notify_all();
1024            while !state.1 {
1025                wake.wait(&mut state);
1026            }
1027        }
1028    }
1029
1030    // Bare state with unconnected shards for pure capacity-accounting tests.
1031    fn state_with_shards(owned: &[usize]) -> PoolState {
1032        let mut state = PoolState::default();
1033
1034        for (id, owned) in owned.iter().enumerate() {
1035            let (tx, _rx) = tokio::sync::mpsc::unbounded_channel();
1036            state.shards.insert(
1037                id,
1038                ShardEntry {
1039                    client: PolymarketWebSocketClient::new_market(
1040                        None,
1041                        false,
1042                        TransportBackend::default(),
1043                    ),
1044                    handle: WsSubscriptionHandle::from_sender(tx),
1045                    forwarder: TaskSlot::new(),
1046                    owned: *owned,
1047                    closing: false,
1048                },
1049            );
1050        }
1051        state
1052    }
1053
1054    fn market_message(filename: &str) -> PolymarketWsMessage {
1055        let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
1056            .join("test_data")
1057            .join(filename);
1058        let json = std::fs::read_to_string(path).unwrap();
1059        PolymarketWsMessage::Market(serde_json::from_str(&json).unwrap())
1060    }
1061
1062    #[rstest]
1063    #[case::primary_new_market("ws_market_new_market_msg.json", true, true)]
1064    #[case::secondary_new_market("ws_market_new_market_msg.json", false, false)]
1065    #[case::primary_resolution("ws_market_resolved_msg.json", true, true)]
1066    #[case::secondary_resolution("ws_market_resolved_msg.json", false, false)]
1067    #[case::secondary_best_bid_ask("ws_market_best_bid_ask_msg.json", false, true)]
1068    fn shard_forwarding_keeps_global_events_on_primary(
1069        #[case] filename: &str,
1070        #[case] is_primary: bool,
1071        #[case] expected: bool,
1072    ) {
1073        let message = market_message(filename);
1074        assert_eq!(should_forward_from_shard(&message, is_primary), expected);
1075    }
1076
1077    #[rstest]
1078    #[tokio::test]
1079    async fn secondary_forwarder_drops_global_events_and_keeps_best_bid_ask() {
1080        let inner = PoolInner::new(None, TransportBackend::default(), true, 1);
1081        let (out_tx, mut out_rx) = tokio::sync::mpsc::unbounded_channel();
1082        *inner.out_tx.lock() = Some(out_tx);
1083        let (shard_tx, shard_rx) = tokio::sync::mpsc::unbounded_channel();
1084        let mut forwarder = inner
1085            .spawn_forwarder(shard_rx, PRIMARY_SHARD_ID + 1)
1086            .expect("spawn forwarder");
1087
1088        shard_tx
1089            .send(market_message("ws_market_new_market_msg.json"))
1090            .unwrap();
1091        shard_tx
1092            .send(market_message("ws_market_resolved_msg.json"))
1093            .unwrap();
1094        shard_tx
1095            .send(market_message("ws_market_best_bid_ask_msg.json"))
1096            .unwrap();
1097        drop(shard_tx);
1098        let outcome = finish_task(
1099            &mut forwarder,
1100            Duration::from_secs(1),
1101            Duration::from_secs(1),
1102        )
1103        .await
1104        .expect("forwarder task");
1105        assert!(matches!(outcome, TaskJoinOutcome::Completed(())));
1106
1107        let forwarded = out_rx.try_recv().unwrap();
1108        let PolymarketWsMessage::Market(MarketWsMessage::BestBidAsk(message)) = forwarded else {
1109            panic!("unexpected forwarded message: {forwarded:?}");
1110        };
1111        assert_eq!(
1112            message.asset_id,
1113            Ustr::from(
1114                "85354956062430465315924116860125388538595433819574542752031640332592237464430"
1115            ),
1116        );
1117        assert!(out_rx.try_recv().is_err());
1118    }
1119
1120    #[rstest]
1121    #[tokio::test]
1122    async fn pool_drain_restores_remaining_state_on_drop() {
1123        let (cmd_tx, _cmd_rx) = tokio::sync::mpsc::unbounded_channel();
1124        let handle = Handle::test_single_shard(cmd_tx, &["token-a"]);
1125        handle
1126            .inner
1127            .state
1128            .lock()
1129            .shutdown_errors
1130            .push("before drain".to_string());
1131
1132        {
1133            let mut drain = PoolDrain::take(&handle.inner.state);
1134            let owner = handle.inner.state.lock();
1135            assert!(owner.shards.is_empty());
1136            assert!(owner.assignments.is_empty());
1137            assert!(owner.shutdown_errors.is_empty());
1138            drop(owner);
1139
1140            assert_eq!(drain.state.shards.len(), 1);
1141            assert_eq!(
1142                drain.state.assignments,
1143                AHashMap::from([(Ustr::from("token-a"), PRIMARY_SHARD_ID)]),
1144            );
1145            assert_eq!(drain.state.shutdown_errors, vec!["before drain"]);
1146            drain.state.shutdown_errors.push("during drain".to_string());
1147        }
1148
1149        let restored = handle.inner.state.lock();
1150        assert_eq!(restored.shards.len(), 1);
1151        let shard = restored
1152            .shards
1153            .get(&PRIMARY_SHARD_ID)
1154            .expect("restored shard");
1155        assert_eq!(shard.owned, 1);
1156        assert!(!shard.closing);
1157        assert_eq!(
1158            restored.assignments,
1159            AHashMap::from([(Ustr::from("token-a"), PRIMARY_SHARD_ID)]),
1160        );
1161        assert_eq!(
1162            restored.shutdown_errors,
1163            vec!["before drain", "during drain"]
1164        );
1165    }
1166
1167    async fn blocked_forwarder() -> (TaskSlot<()>, std::sync::mpsc::Sender<()>) {
1168        let (release, blocked) = std::sync::mpsc::channel();
1169        let (started, ready) = tokio::sync::oneshot::channel();
1170        let handle = tokio::task::spawn_blocking(move || {
1171            let _ = started.send(());
1172            let _ = blocked.recv();
1173        });
1174        ready.await.unwrap();
1175        (TaskSlot::from_handle(handle), release)
1176    }
1177
1178    #[rstest]
1179    #[case::single(1)]
1180    #[case::many(64)]
1181    #[tokio::test(start_paused = true)]
1182    async fn pool_disconnects_concurrently_and_preserves_failures(#[case] count: usize) {
1183        let pool = PolymarketMarketConnectionPool::new(None, false, TransportBackend::default(), 1);
1184        let mut state = state_with_shards(&vec![1; count]);
1185        let mut releases = Vec::new();
1186
1187        for (&id, shard) in &mut state.shards {
1188            let (forwarder, release) = blocked_forwarder().await;
1189            shard.forwarder = forwarder;
1190            releases.push(release);
1191            state.assignments.insert(Ustr::from(&id.to_string()), id);
1192        }
1193        *pool.inner.state.lock() = state;
1194
1195        let started = tokio::time::Instant::now();
1196        let expected = Duration::from_secs(2);
1197        let mut disconnect = std::pin::pin!(pool.disconnect());
1198        let result = loop {
1199            if let std::task::Poll::Ready(result) = futures_util::poll!(disconnect.as_mut()) {
1200                break result;
1201            }
1202            assert!(started.elapsed() < expected + Duration::from_secs(1));
1203            tokio::time::advance(Duration::from_millis(100)).await;
1204        };
1205        assert!(started.elapsed() >= expected);
1206        let error = result.unwrap_err().to_string();
1207        let failures = error.matches("forwarder did not stop after abort");
1208        assert_eq!(failures.count(), count);
1209        assert_eq!(pool.connection_count(), count);
1210        assert_eq!(pool.subscription_count(), count);
1211
1212        drop(releases);
1213        pool.disconnect().await.expect("cleanup retry");
1214        assert_eq!(pool.connection_count(), 0);
1215        assert_eq!(pool.subscription_count(), 0);
1216    }
1217
1218    #[rstest]
1219    #[tokio::test]
1220    async fn canceled_pool_disconnect_restores_only_unfinished_ownership() {
1221        let pool = PolymarketMarketConnectionPool::new(None, false, TransportBackend::default(), 1);
1222        let mut state = state_with_shards(&[1, 1]);
1223        let (forwarder, release) = blocked_forwarder().await;
1224        state.shards.get_mut(&1).unwrap().forwarder = forwarder;
1225        state.assignments = AHashMap::from([(Ustr::from("done"), 0), (Ustr::from("pending"), 1)]);
1226        *pool.inner.state.lock() = state;
1227
1228        {
1229            let mut disconnect = std::pin::pin!(pool.disconnect());
1230            assert!(futures_util::poll!(disconnect.as_mut()).is_pending());
1231        }
1232        assert_eq!(pool.connection_count(), 1);
1233        assert_eq!(
1234            pool.inner.state.lock().assignments,
1235            AHashMap::from([(Ustr::from("pending"), 1)])
1236        );
1237
1238        drop(release);
1239        pool.disconnect().await.expect("cleanup retry");
1240        assert_eq!(pool.connection_count(), 0);
1241        assert_eq!(pool.subscription_count(), 0);
1242    }
1243
1244    #[rstest]
1245    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1246    async fn canceled_shard_close_restores_unfinished_ownership() {
1247        let inner = Arc::new(PoolInner::new(None, TransportBackend::default(), false, 1));
1248        let blocking = StdArc::new((TestMutex::new((false, false)), Condvar::new()));
1249        let blocking_task = StdArc::clone(&blocking);
1250        let (started_tx, started_rx) = tokio::sync::oneshot::channel();
1251        let mut forwarder = TaskSlot::new();
1252        forwarder
1253            .spawn(async move {
1254                let _blocking = BlockingDrop(blocking_task);
1255                let _ = started_tx.send(());
1256                std::future::pending::<()>().await;
1257            })
1258            .expect("spawn forwarder");
1259        started_rx.await.expect("forwarder should start");
1260
1261        let (cmd_tx, _cmd_rx) = tokio::sync::mpsc::unbounded_channel();
1262        let shard = Box::new(ShardEntry {
1263            client: PolymarketWebSocketClient::new_market(None, false, TransportBackend::default()),
1264            handle: WsSubscriptionHandle::from_sender(cmd_tx),
1265            forwarder,
1266            owned: 0,
1267            closing: false,
1268        });
1269        let close_inner = Arc::clone(&inner);
1270        let close = tokio::spawn(async move {
1271            let _result = close_inner.close_shard(1, shard).await;
1272        });
1273
1274        loop {
1275            if blocking.0.lock().0 {
1276                break;
1277            }
1278            tokio::task::yield_now().await;
1279        }
1280        close.abort();
1281        let _ = close.await;
1282
1283        {
1284            let restored = inner.state.lock();
1285            let shard = restored.shards.get(&1).expect("closing shard restored");
1286            assert!(shard.closing);
1287            assert!(shard.forwarder.is_some());
1288            assert_eq!(smallest_shard_with_capacity(&restored, 1), None);
1289        }
1290
1291        {
1292            let (state, wake) = &*blocking;
1293            state.lock().1 = true;
1294            wake.notify_all();
1295        }
1296        let shard = inner
1297            .state
1298            .lock()
1299            .shards
1300            .remove(&1)
1301            .expect("restored shard");
1302        inner
1303            .close_shard(1, Box::new(shard))
1304            .await
1305            .expect("restored shard should close");
1306
1307        assert!(inner.state.lock().shards.is_empty());
1308    }
1309
1310    #[rstest]
1311    fn zero_max_subscriptions_clamps_to_default() {
1312        let inner = PoolInner::new(None, TransportBackend::default(), false, 0);
1313        assert_eq!(inner.max_subscriptions, WS_DEFAULT_SUBSCRIPTIONS);
1314    }
1315
1316    #[rstest]
1317    #[tokio::test]
1318    async fn secondary_market_connection_timeout() {
1319        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
1320        let url = format!("ws://{}", listener.local_addr().unwrap());
1321
1322        let server = tokio::spawn(async move {
1323            let mut connections = Vec::new();
1324
1325            loop {
1326                let (stream, _) = listener.accept().await.unwrap();
1327                connections.push(stream);
1328            }
1329        });
1330
1331        let pool =
1332            PolymarketMarketConnectionPool::new(Some(url), false, TransportBackend::default(), 1)
1333                .with_connect_timeout(1);
1334        let mut ws_client = pool.inner.market_client(false, PRIMARY_SHARD_ID + 1);
1335
1336        let error = tokio::time::timeout(Duration::from_secs(20), ws_client.connect())
1337            .await
1338            .expect("secondary shard must use the configured connection timeout")
1339            .unwrap_err();
1340        server.abort();
1341
1342        assert!(
1343            matches!(error.downcast_ref::<TransportError>(),
1344                Some(TransportError::Io(e)) if e.kind() == std::io::ErrorKind::TimedOut),
1345            "{error:#}",
1346        );
1347    }
1348
1349    #[rstest]
1350    fn pool_retains_proxy_for_lazily_created_shards() {
1351        const PROXY_URL: &str = "http://pool-user:pool-proxy-secret@127.0.0.1:18088";
1352        let pool = PolymarketMarketConnectionPool::new_with_proxy(
1353            Some("ws://market.example/ws".to_string()),
1354            true,
1355            TransportBackend::Tungstenite,
1356            17,
1357            Some(ProxyUrl::parse(PROXY_URL).unwrap()),
1358        );
1359        let primary = pool.inner.market_client(true, PRIMARY_SHARD_ID);
1360        let secondary = pool.inner.market_client(false, PRIMARY_SHARD_ID + 1);
1361        let debug = format!("{pool:?}");
1362
1363        assert_eq!(pool.inner.proxy_url.as_ref().unwrap().expose(), PROXY_URL);
1364        assert_eq!(primary.proxy_url().unwrap().expose(), PROXY_URL);
1365        assert_eq!(secondary.proxy_url().unwrap().expose(), PROXY_URL);
1366        assert_eq!(pool.inner.max_subscriptions, 17);
1367        assert!(!debug.contains("pool-proxy-secret"));
1368    }
1369
1370    #[rstest]
1371    #[case::first_has_room(&[0, 200], 200, Some(0))]
1372    #[case::prefers_lowest_id(&[200, 5, 5], 200, Some(1))]
1373    #[case::all_full(&[200, 200], 200, None)]
1374    #[case::exact_boundary_is_full(&[1], 1, None)]
1375    fn smallest_shard_with_capacity_picks_lowest_open_id(
1376        #[case] owned: &[usize],
1377        #[case] max: usize,
1378        #[case] expected: Option<usize>,
1379    ) {
1380        let state = state_with_shards(owned);
1381        assert_eq!(smallest_shard_with_capacity(&state, max), expected);
1382    }
1383
1384    #[rstest]
1385    fn available_shard_id_reuses_lowest_closed_shard() {
1386        let mut state = state_with_shards(&[1, 1, 1]);
1387        state.shards.remove(&(PRIMARY_SHARD_ID + 1));
1388
1389        assert_eq!(available_shard_id(&state), PRIMARY_SHARD_ID + 1);
1390    }
1391
1392    #[rstest]
1393    #[tokio::test]
1394    async fn subscribe_routes_command_and_tracks_assignment() {
1395        let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
1396        let handle = Handle::test_single_shard(tx, &[]);
1397
1398        handle
1399            .subscribe_market(vec!["token-a".to_string()])
1400            .await
1401            .expect("subscribe");
1402
1403        match rx.try_recv().expect("expected SubscribeMarket") {
1404            HandlerCommand::SubscribeMarket(ids) => assert_eq!(ids, vec!["token-a".to_string()]),
1405            other => panic!("unexpected command: {other:?}"),
1406        }
1407        assert_eq!(handle.inner.subscription_count_for_test(), 1);
1408    }
1409
1410    #[rstest]
1411    #[tokio::test]
1412    async fn duplicate_subscribe_does_not_consume_capacity_or_resend() {
1413        let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
1414        let handle = Handle::test_single_shard(tx, &[]);
1415
1416        handle
1417            .subscribe_market(vec!["token-a".to_string()])
1418            .await
1419            .expect("first subscribe");
1420        handle
1421            .subscribe_market(vec!["token-a".to_string()])
1422            .await
1423            .expect("duplicate subscribe");
1424
1425        assert!(matches!(
1426            rx.try_recv(),
1427            Ok(HandlerCommand::SubscribeMarket(_))
1428        ));
1429        assert!(rx.try_recv().is_err(), "duplicate must not resend");
1430        assert_eq!(handle.inner.subscription_count_for_test(), 1);
1431    }
1432
1433    #[rstest]
1434    #[tokio::test]
1435    async fn unsubscribe_routes_command_and_releases_assignment() {
1436        let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
1437        let handle = Handle::test_single_shard(tx, &["token-a"]);
1438
1439        handle
1440            .unsubscribe_market(vec!["token-a".to_string()])
1441            .await
1442            .expect("unsubscribe");
1443
1444        match rx.try_recv().expect("expected UnsubscribeMarket") {
1445            HandlerCommand::UnsubscribeMarket(ids) => assert_eq!(ids, vec!["token-a".to_string()]),
1446            other => panic!("unexpected command: {other:?}"),
1447        }
1448        assert_eq!(handle.inner.subscription_count_for_test(), 0);
1449    }
1450
1451    #[rstest]
1452    #[tokio::test]
1453    async fn unsubscribe_unknown_token_is_noop() {
1454        let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
1455        let handle = Handle::test_single_shard(tx, &[]);
1456
1457        handle
1458            .unsubscribe_market(vec!["token-a".to_string()])
1459            .await
1460            .expect("unsubscribe");
1461
1462        assert!(rx.try_recv().is_err());
1463    }
1464
1465    #[rstest]
1466    #[tokio::test]
1467    async fn subscribe_send_failure_rolls_back_assignment() {
1468        let (tx, rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
1469        drop(rx);
1470        let handle = Handle::test_single_shard(tx, &[]);
1471
1472        let result = handle.subscribe_market(vec!["token-a".to_string()]).await;
1473
1474        assert!(result.is_err());
1475        assert_eq!(handle.inner.subscription_count_for_test(), 0);
1476    }
1477
1478    #[rstest]
1479    #[tokio::test]
1480    async fn resubscribe_market_cycles_without_changing_assignment() {
1481        let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
1482        let handle = Handle::test_single_shard(tx, &["token-a"]);
1483        let cancel = CancellationToken::new();
1484
1485        let task = {
1486            let handle = handle.clone();
1487            let cancel = cancel.clone();
1488
1489            tokio::spawn(async move {
1490                handle
1491                    .resubscribe_market(
1492                        vec!["token-a".to_string()],
1493                        &cancel,
1494                        &SnapshotGate::default(),
1495                    )
1496                    .await
1497            })
1498        };
1499
1500        match rx.recv().await.expect("expected cycle command") {
1501            HandlerCommand::CycleMarketSubscription {
1502                asset_ids,
1503                responder,
1504                ..
1505            } => {
1506                assert_eq!(asset_ids, vec!["token-a".to_string()]);
1507                responder
1508                    .send(CycleMarketOutcome::Completed)
1509                    .expect("answer cycle");
1510            }
1511            other => panic!("unexpected command: {other:?}"),
1512        }
1513
1514        assert!(matches!(
1515            task.await.expect("cycle task").expect("cycle owned asset"),
1516            CycleMarketOutcome::Completed
1517        ));
1518        assert_eq!(
1519            handle.tokens_for_shard(PRIMARY_SHARD_ID),
1520            vec![Ustr::from("token-a")]
1521        );
1522        assert!(handle.tokens_for_shard(PRIMARY_SHARD_ID + 1).is_empty());
1523        assert_eq!(handle.inner.subscription_count_for_test(), 1);
1524    }
1525
1526    #[rstest]
1527    #[tokio::test]
1528    async fn resubscribe_market_propagates_cycle_outcome() {
1529        let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
1530        let handle = Handle::test_single_shard(tx, &["token-a"]);
1531        let cancel = CancellationToken::new();
1532
1533        let task = {
1534            let handle = handle.clone();
1535            let cancel = cancel.clone();
1536
1537            tokio::spawn(async move {
1538                handle
1539                    .resubscribe_market(
1540                        vec!["token-a".to_string()],
1541                        &cancel,
1542                        &SnapshotGate::default(),
1543                    )
1544                    .await
1545            })
1546        };
1547
1548        match rx.recv().await.expect("expected cycle command") {
1549            HandlerCommand::CycleMarketSubscription { responder, .. } => {
1550                responder
1551                    .send(CycleMarketOutcome::ConnectionChanged)
1552                    .expect("answer cycle");
1553            }
1554            other => panic!("unexpected command: {other:?}"),
1555        }
1556
1557        assert!(matches!(
1558            task.await.expect("cycle task").expect("cycle reports"),
1559            CycleMarketOutcome::ConnectionChanged
1560        ));
1561        // Ownership is preserved even when the cycle does not complete.
1562        assert_eq!(
1563            handle.tokens_for_shard(PRIMARY_SHARD_ID),
1564            vec![Ustr::from("token-a")]
1565        );
1566    }
1567
1568    #[rstest]
1569    #[tokio::test]
1570    async fn resubscribe_market_fails_when_cycle_response_lost() {
1571        let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
1572        let handle = Handle::test_single_shard(tx, &["token-a"]);
1573        let cancel = CancellationToken::new();
1574
1575        let task = {
1576            let handle = handle.clone();
1577            let cancel = cancel.clone();
1578
1579            tokio::spawn(async move {
1580                handle
1581                    .resubscribe_market(
1582                        vec!["token-a".to_string()],
1583                        &cancel,
1584                        &SnapshotGate::default(),
1585                    )
1586                    .await
1587            })
1588        };
1589
1590        // Simulate handler death: receive the command, drop the responder.
1591        assert!(matches!(
1592            rx.recv().await.expect("expected cycle command"),
1593            HandlerCommand::CycleMarketSubscription { .. }
1594        ));
1595
1596        let err = task.await.expect("cycle task").expect_err("lost response");
1597
1598        assert!(
1599            err.to_string().contains("response lost"),
1600            "unexpected error: {err}"
1601        );
1602    }
1603
1604    #[rstest]
1605    #[tokio::test]
1606    async fn resubscribe_market_rejects_unowned_asset() {
1607        let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
1608        let handle = Handle::test_single_shard(tx, &["token-a"]);
1609
1610        let err = handle
1611            .resubscribe_market(
1612                vec!["token-unknown".to_string()],
1613                &CancellationToken::new(),
1614                &SnapshotGate::default(),
1615            )
1616            .await
1617            .expect_err("unowned asset must fail");
1618
1619        assert!(
1620            err.to_string().contains("not owned by any shard"),
1621            "unexpected error: {err}"
1622        );
1623        assert!(rx.try_recv().is_err());
1624    }
1625
1626    #[rstest]
1627    #[tokio::test]
1628    async fn resubscribe_market_rejects_missing_shard() {
1629        let (tx, _rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
1630        let handle = Handle::test_single_shard(tx, &["token-a"]);
1631        handle.inner.state.lock().shards.remove(&PRIMARY_SHARD_ID);
1632
1633        let err = handle
1634            .resubscribe_market(
1635                vec!["token-a".to_string()],
1636                &CancellationToken::new(),
1637                &SnapshotGate::default(),
1638            )
1639            .await
1640            .expect_err("missing shard must fail");
1641
1642        assert!(
1643            err.to_string().contains("is not available"),
1644            "unexpected error: {err}"
1645        );
1646    }
1647
1648    #[rstest]
1649    #[tokio::test]
1650    async fn resubscribe_market_rejects_closed_pool() {
1651        let (tx, _rx) = tokio::sync::mpsc::unbounded_channel::<HandlerCommand>();
1652        let handle = Handle::test_single_shard(tx, &["token-a"]);
1653        handle.inner.closed.store(true, Ordering::SeqCst);
1654
1655        let err = handle
1656            .resubscribe_market(
1657                vec!["token-a".to_string()],
1658                &CancellationToken::new(),
1659                &SnapshotGate::default(),
1660            )
1661            .await
1662            .expect_err("closed pool must fail");
1663
1664        assert!(
1665            err.to_string().contains("closed"),
1666            "unexpected error: {err}"
1667        );
1668    }
1669
1670    #[rstest]
1671    #[tokio::test]
1672    async fn forwarder_stamps_reconnected_with_shard_id() {
1673        let inner = PoolInner::new(None, TransportBackend::default(), true, 1);
1674        let (out_tx, mut out_rx) = tokio::sync::mpsc::unbounded_channel();
1675        *inner.out_tx.lock() = Some(out_tx);
1676        let (shard_tx, shard_rx) = tokio::sync::mpsc::unbounded_channel();
1677        let mut forwarder = inner
1678            .spawn_forwarder(shard_rx, PRIMARY_SHARD_ID + 1)
1679            .expect("spawn forwarder");
1680
1681        shard_tx
1682            .send(PolymarketWsMessage::Reconnected { shard_id: None })
1683            .unwrap();
1684        shard_tx
1685            .send(market_message("ws_market_best_bid_ask_msg.json"))
1686            .unwrap();
1687        drop(shard_tx);
1688        let outcome = finish_task(
1689            &mut forwarder,
1690            Duration::from_secs(1),
1691            Duration::from_secs(1),
1692        )
1693        .await
1694        .expect("forwarder task");
1695        assert!(matches!(outcome, TaskJoinOutcome::Completed(())));
1696
1697        assert!(
1698            matches!(
1699                out_rx.try_recv().unwrap(),
1700                PolymarketWsMessage::Reconnected { shard_id: Some(id) }
1701                if id == PRIMARY_SHARD_ID + 1
1702            ),
1703            "forwarder must stamp the reconnecting shard"
1704        );
1705        assert!(matches!(
1706            out_rx.try_recv().unwrap(),
1707            PolymarketWsMessage::Market(MarketWsMessage::BestBidAsk(_))
1708        ));
1709        assert!(out_rx.try_recv().is_err());
1710    }
1711}