Skip to main content

nautilus_blockchain/hypersync/
client.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16use std::{
17    env::VarError,
18    sync::{
19        Arc,
20        atomic::{AtomicU64, Ordering},
21    },
22    time::Duration,
23};
24
25use ahash::AHashMap;
26use alloy::primitives::Address;
27use anyhow::Context;
28use futures_util::Stream;
29use hypersync_client::{
30    StreamConfig,
31    net_types::{BlockField, BlockSelection, FieldSelection, Query},
32    simple_types::Log,
33};
34use nautilus_common::config::{ConfigError, check_non_empty_field};
35use nautilus_core::{consts::NAUTILUS_USER_AGENT, hex};
36use nautilus_live::task::{TaskJoinOutcome, TaskSlot, finish_task};
37use nautilus_model::{
38    defi::{Block, Blockchain, DexType, SharedChain},
39    identifiers::InstrumentId,
40};
41
42use crate::{
43    exchanges::{extended::DexExtended, get_dex_extended},
44    hypersync::transform::transform_hypersync_block,
45    rpc::{http::validate_execution_endpoint, types::BlockchainMessage},
46};
47
48const API_TOKEN_ENV: &str = "ENVIO_API_TOKEN";
49const API_TOKEN_FORMAT: &str = "a UUID";
50
51/// An item yielded by the contract-events stream.
52///
53/// Blocks are surfaced ahead of the logs from the same response so callers can populate their
54/// block-timestamp cache before converting events from those blocks.
55#[derive(Debug)]
56pub enum PoolEventStreamItem {
57    /// A block referenced by subsequent logs.
58    Block(Block),
59    /// A contract event log.
60    Log(Log),
61}
62
63/// The interval in milliseconds at which to check for new blocks when waiting
64/// for the hypersync to index the block.
65const BLOCK_POLLING_INTERVAL_MS: u64 = 50;
66
67/// Timeout in seconds for HyperSync HTTP requests.
68const HYPERSYNC_REQUEST_TIMEOUT_SECS: u64 = 30;
69
70/// Timeout in seconds for graceful task shutdown during disconnect.
71/// If the task doesn't finish within this time, it will be forcefully aborted.
72const DISCONNECT_TIMEOUT_SECS: u64 = 5;
73
74/// Delay before restarting a DEX event stream after it reaches the current indexed tip.
75const DEX_EVENT_STREAM_RETRY_DELAY_MS: u64 = 1_000;
76
77/// A client for interacting with a HyperSync API to retrieve blockchain data.
78#[derive(Debug)]
79pub struct HyperSyncClient {
80    /// The target blockchain identifier (e.g. Ethereum, Arbitrum).
81    chain: SharedChain,
82    /// The underlying HyperSync Rust client for making API requests.
83    client: Arc<hypersync_client::Client>,
84    /// Background task handle for the block subscription task.
85    blocks_task: TaskSlot<()>,
86    /// Cancellation token for the blocks subscription task.
87    blocks_cancellation_token: Option<tokio_util::sync::CancellationToken>,
88    /// Background DEX event stream tasks keyed by DEX type.
89    dex_event_tasks: AHashMap<DexType, DexEventStreamTask>,
90    /// Channel for sending blockchain messages to the adapter data client.
91    tx: Option<tokio::sync::mpsc::UnboundedSender<BlockchainMessage>>,
92    /// Index of pool addressed keyed by instrument ID.
93    pool_addresses: AHashMap<InstrumentId, Address>,
94    /// Cancellation token for graceful shutdown of background tasks.
95    cancellation_token: tokio_util::sync::CancellationToken,
96}
97
98impl HyperSyncClient {
99    /// Creates a new [`HyperSyncClient`] instance for the given chain and message sender.
100    ///
101    /// # Errors
102    ///
103    /// Returns an error if the HyperSync endpoint or `ENVIO_API_TOKEN` is invalid, the token
104    /// is missing, or the underlying client cannot be initialized.
105    pub fn new(
106        chain: SharedChain,
107        tx: Option<tokio::sync::mpsc::UnboundedSender<BlockchainMessage>>,
108        cancellation_token: tokio_util::sync::CancellationToken,
109    ) -> anyhow::Result<Self> {
110        let mut config = hypersync_client::ClientConfig::default();
111
112        let hypersync_url = chain
113            .hypersync_url
114            .as_deref()
115            .ok_or_else(|| ConfigError::missing_field("chain.hypersync_url"))?;
116        let hypersync_url = validate_execution_endpoint(hypersync_url, "HyperSync")
117            .map_err(|e| ConfigError::invalid_value("chain.hypersync_url", e.to_string()))?;
118
119        config.url = hypersync_url.to_string();
120        config.api_token = std::env::var(API_TOKEN_ENV).map_err(|e| match e {
121            VarError::NotPresent => ConfigError::missing_field(API_TOKEN_ENV),
122            VarError::NotUnicode(_) => ConfigError::invalid_format(API_TOKEN_ENV, API_TOKEN_FORMAT),
123        })?;
124
125        check_non_empty_field(API_TOKEN_ENV, &config.api_token)?;
126
127        config
128            .validate()
129            .map_err(|_| ConfigError::invalid_format(API_TOKEN_ENV, API_TOKEN_FORMAT))?;
130        let client = hypersync_client::Client::new_with_agent(config, NAUTILUS_USER_AGENT)
131            .context("failed to create HyperSync client")?;
132
133        Ok(Self {
134            chain,
135            client: Arc::new(client),
136            blocks_task: TaskSlot::new(),
137            blocks_cancellation_token: None,
138            dex_event_tasks: AHashMap::new(),
139            tx,
140            pool_addresses: AHashMap::new(),
141            cancellation_token,
142        })
143    }
144
145    #[must_use]
146    pub fn get_pool_address(&self, instrument_id: InstrumentId) -> Option<&Address> {
147        self.pool_addresses.get(&instrument_id)
148    }
149
150    /// Starts, refreshes, or stops the live DEX event stream for one DEX.
151    ///
152    /// The stream query is open-ended (`to_block = None`) and the background task resumes from the
153    /// last HyperSync `next_block` whenever the SDK stream reaches the current indexed tip.
154    pub async fn update_dex_event_stream(
155        &mut self,
156        dex: DexType,
157        contract_addresses: Vec<Address>,
158        event_signatures: Vec<String>,
159    ) {
160        let filter = DexEventStreamFilter::new(contract_addresses, event_signatures);
161
162        if filter.is_empty() {
163            self.stop_dex_event_stream(dex).await;
164            return;
165        }
166
167        if self.dex_event_tasks.get(&dex).is_some_and(|task| {
168            task.filter == filter
169                && task
170                    .task
171                    .as_ref()
172                    .is_some_and(|handle| !handle.is_finished())
173        }) {
174            return;
175        }
176
177        let next_from_block = self.stop_dex_event_stream(dex).await;
178        if self.dex_event_tasks.contains_key(&dex) {
179            log::error!("Previous HyperSync DEX event stream for {dex} is still stopping");
180            return;
181        }
182
183        let from_block = match next_from_block {
184            Some(block) => block,
185            None => match self.client.get_height().await {
186                Ok(block) => block,
187                Err(e) => {
188                    log::error!("Failed to get HyperSync height for DEX event stream: {e}");
189                    return;
190                }
191            },
192        };
193
194        let tx = if let Some(tx) = &self.tx {
195            tx.clone()
196        } else {
197            log::error!("Hypersync client channel should have been initialized");
198            return;
199        };
200
201        let client = Arc::clone(&self.client);
202        let chain = self.chain.name;
203        let Some(dex_extended) = get_dex_extended(chain, &dex) else {
204            log::error!("Failed to get DEX registration for {dex} on {chain}");
205            return;
206        };
207        let stream_token = self.cancellation_token.child_token();
208        let task_token = stream_token.clone();
209        let next_from_block = Arc::new(AtomicU64::new(from_block));
210        let task_next_from_block = Arc::clone(&next_from_block);
211        let task_filter = filter.clone();
212
213        let mut task = TaskSlot::new();
214        if let Err(e) = task.spawn(async move {
215            Self::run_dex_event_stream(
216                dex,
217                client,
218                tx,
219                task_filter,
220                dex_extended,
221                task_next_from_block,
222                task_token,
223            )
224            .await;
225        }) {
226            stream_token.cancel();
227            log::error!("Failed to start HyperSync DEX event stream for {dex}: {e}");
228        }
229
230        self.dex_event_tasks.insert(
231            dex,
232            DexEventStreamTask {
233                filter,
234                next_from_block,
235                cancellation_token: stream_token,
236                task,
237            },
238        );
239    }
240
241    /// Creates a stream of contract event logs matching the specified criteria.
242    ///
243    /// # Panics
244    ///
245    /// Panics if the contract address cannot be parsed as a valid Ethereum address.
246    pub async fn request_contract_events_stream(
247        &self,
248        from_block: u64,
249        to_block: Option<u64>,
250        contract_address: &Address,
251        topics: Vec<&str>,
252    ) -> impl Stream<Item = PoolEventStreamItem> + use<> {
253        let query = Self::construct_contract_events_query(
254            from_block,
255            to_block,
256            &[*contract_address],
257            &topics,
258        );
259
260        let chain = self.chain.name;
261        let mut rx = self
262            .client
263            .stream(query, StreamConfig::default())
264            .await
265            .expect("Failed to create stream");
266
267        async_stream::stream! {
268              while let Some(response) = rx.recv().await {
269                let response = response.unwrap();
270                for item in pool_events_from_response(chain, response.data.blocks, response.data.logs) {
271                    yield item;
272                }
273            }
274        }
275    }
276
277    /// Disconnects from the HyperSync service and stops all background tasks.
278    pub async fn disconnect(&mut self) {
279        log::debug!("Disconnecting HyperSync client");
280        self.cancellation_token.cancel();
281
282        if let Some(outcome) = finish_task(
283            &mut self.blocks_task,
284            Duration::from_secs(DISCONNECT_TIMEOUT_SECS),
285            Duration::from_secs(DISCONNECT_TIMEOUT_SECS),
286        )
287        .await
288        {
289            match outcome {
290                TaskJoinOutcome::Completed(()) | TaskJoinOutcome::Aborted => {}
291                TaskJoinOutcome::Failed(error) => {
292                    log::error!("HyperSync blocks task failed: {error}");
293                }
294                TaskJoinOutcome::Incomplete => {
295                    log::error!("HyperSync blocks task did not stop after abort");
296                }
297            }
298        }
299
300        let dexes = self.dex_event_tasks.keys().copied().collect::<Vec<_>>();
301        for dex in dexes {
302            self.stop_dex_event_stream(dex).await;
303        }
304
305        log::debug!("HyperSync client disconnected");
306    }
307
308    /// Returns the current block
309    ///
310    /// # Panics
311    ///
312    /// Panics if the client height request fails.
313    pub async fn current_block(&self) -> u64 {
314        self.client.get_height().await.unwrap()
315    }
316
317    /// Creates a stream that yields blockchain blocks within the specified range.
318    ///
319    /// # Panics
320    ///
321    /// Panics if the stream creation or block transformation fails.
322    pub async fn request_blocks_stream(
323        &self,
324        from_block: u64,
325        to_block: Option<u64>,
326    ) -> impl Stream<Item = Block> {
327        let query = Self::construct_block_query(from_block, to_block);
328        let mut rx = self
329            .client
330            .stream(query, StreamConfig::default())
331            .await
332            .unwrap();
333
334        let chain = self.chain.name;
335
336        async_stream::stream! {
337            while let Some(response) = rx.recv().await {
338                let response = response.unwrap();
339                for received_block in response.data.blocks.into_iter().flatten() {
340                    let block = transform_hypersync_block(chain, received_block).unwrap();
341                    yield block
342                }
343            }
344        }
345    }
346
347    /// Starts a background task that continuously polls for new blockchain blocks.
348    ///
349    /// # Panics
350    ///
351    /// Panics if client height requests or block transformations fail.
352    pub fn subscribe_blocks(&mut self) {
353        if self.blocks_task.is_some() {
354            return;
355        }
356
357        let chain = self.chain.name;
358        let client = Arc::clone(&self.client);
359        let tx = if let Some(tx) = &self.tx {
360            tx.clone()
361        } else {
362            log::error!("Hypersync client channel should have been initialized");
363            return;
364        };
365
366        // Create a child token that can be cancelled independently
367        let blocks_token = self.cancellation_token.child_token();
368        let cancellation_token = blocks_token.clone();
369        self.blocks_cancellation_token = Some(blocks_token);
370
371        if let Err(e) = self.blocks_task.spawn(async move {
372            log::debug!("Starting task 'blocks_feed");
373
374            let current_block_height = client.get_height().await.unwrap();
375            let mut query = Self::construct_block_query(current_block_height, None);
376
377            loop {
378                tokio::select! {
379                    () = cancellation_token.cancelled() => {
380                        log::debug!("Blocks subscription task received cancellation signal");
381                        break;
382                    }
383                    result = tokio::time::timeout(
384                        Duration::from_secs(HYPERSYNC_REQUEST_TIMEOUT_SECS),
385                        client.get(&query)
386                    ) => {
387                        let response = match result {
388                            Ok(Ok(resp)) => resp,
389                            Ok(Err(e)) => {
390                                log::error!("Hypersync request failed: {e}");
391                                break;
392                            }
393                            Err(_) => {
394                                log::warn!("Hypersync request timed out after {HYPERSYNC_REQUEST_TIMEOUT_SECS}s, retrying...");
395                                continue;
396                            }
397                        };
398
399                        for received_block in response.data.blocks.into_iter().flatten() {
400                            let block = transform_hypersync_block(chain, received_block).unwrap();
401                            let msg = BlockchainMessage::Block(block);
402                            if let Err(e) = tx.send(msg) {
403                                log::error!("Error sending message: {e}");
404                            }
405                        }
406
407                        if let Some(archive_block_height) = response.archive_height
408                            && archive_block_height < response.next_block
409                        {
410                            while client.get_height().await.unwrap() < response.next_block {
411                                tokio::select! {
412                                    () = cancellation_token.cancelled() => {
413                                        log::debug!("Blocks subscription task received cancellation signal during polling");
414                                        return;
415                                    }
416                                    () = tokio::time::sleep(Duration::from_millis(
417                                        BLOCK_POLLING_INTERVAL_MS,
418                                    )) => {}
419                                }
420                            }
421                        }
422
423                        query.from_block = response.next_block;
424                    }
425                }
426            }
427        }) {
428            if let Some(token) = self.blocks_cancellation_token.take() {
429                token.cancel();
430            }
431            log::error!("Failed to start HyperSync blocks subscription task: {e}");
432        }
433    }
434
435    /// Unsubscribes from new blocks by stopping the background watch task.
436    pub async fn unsubscribe_blocks(&mut self) {
437        if self.blocks_task.is_none() {
438            return;
439        }
440
441        // Cancel only the blocks child token, not the main cancellation token
442        if let Some(token) = self.blocks_cancellation_token.take() {
443            token.cancel();
444        }
445
446        if let Some(outcome) = finish_task(
447            &mut self.blocks_task,
448            Duration::from_secs(DISCONNECT_TIMEOUT_SECS),
449            Duration::from_secs(DISCONNECT_TIMEOUT_SECS),
450        )
451        .await
452        {
453            match outcome {
454                TaskJoinOutcome::Completed(()) | TaskJoinOutcome::Aborted => {}
455                TaskJoinOutcome::Failed(error) => {
456                    log::error!("HyperSync blocks task failed during unsubscribe: {error}");
457                }
458                TaskJoinOutcome::Incomplete => {
459                    log::error!("HyperSync blocks task did not stop after abort");
460                }
461            }
462        }
463        log::debug!("Unsubscribed from blocks");
464    }
465
466    /// Constructs a HyperSync query for fetching blocks with all available fields within the specified range.
467    fn construct_block_query(from_block: u64, to_block: Option<u64>) -> Query {
468        Query {
469            from_block,
470            to_block: Self::to_hypersync_exclusive_bound(to_block),
471            blocks: vec![BlockSelection::default()],
472            field_selection: FieldSelection {
473                block: BlockField::all(),
474                ..Default::default()
475            },
476            ..Default::default()
477        }
478    }
479
480    fn construct_contract_events_query(
481        from_block: u64,
482        to_block: Option<u64>,
483        contract_addresses: &[Address],
484        topics: &[&str],
485    ) -> Query {
486        let mut query_value = serde_json::json!({
487            "from_block": from_block,
488            "logs": [{
489                "topics": [topics],
490                "address": contract_addresses
491            }],
492            "field_selection": {
493                "log": [
494                    "block_number",
495                    "transaction_hash",
496                    "transaction_index",
497                    "log_index",
498                    "address",
499                    "data",
500                    "topic0",
501                    "topic1",
502                    "topic2",
503                    "topic3",
504                ],
505                // Join block fields so callers can resolve each event's ts_event
506                "block": [
507                    "number",
508                    "hash",
509                    "parent_hash",
510                    "miner",
511                    "gas_limit",
512                    "gas_used",
513                    "timestamp",
514                ]
515            }
516        });
517
518        if let Some(to_block) = Self::to_hypersync_exclusive_bound(to_block)
519            && let Some(obj) = query_value.as_object_mut()
520        {
521            obj.insert("to_block".to_string(), serde_json::json!(to_block));
522        }
523
524        serde_json::from_value(query_value).unwrap()
525    }
526
527    fn to_hypersync_exclusive_bound(to_block: Option<u64>) -> Option<u64> {
528        to_block.map(|block| block.saturating_add(1))
529    }
530
531    async fn run_dex_event_stream(
532        dex: DexType,
533        client: Arc<hypersync_client::Client>,
534        tx: tokio::sync::mpsc::UnboundedSender<BlockchainMessage>,
535        filter: DexEventStreamFilter,
536        dex_extended: &'static DexExtended,
537        next_from_block: Arc<AtomicU64>,
538        cancellation_token: tokio_util::sync::CancellationToken,
539    ) {
540        log::debug!("Starting task 'dex_event_stream' for {dex}");
541
542        loop {
543            let from_block = next_from_block.load(Ordering::Relaxed);
544            Self::wait_for_stream_start_block(&client, from_block, &cancellation_token).await;
545            if cancellation_token.is_cancelled() {
546                break;
547            }
548
549            let topics = filter
550                .event_signatures
551                .iter()
552                .map(String::as_str)
553                .collect::<Vec<_>>();
554            let query = Self::construct_contract_events_query(
555                from_block,
556                None,
557                &filter.contract_addresses,
558                &topics,
559            );
560            let mut rx = match client.stream(query, StreamConfig::default()).await {
561                Ok(rx) => rx,
562                Err(e) => {
563                    log::warn!("Failed to create DEX event stream for {dex}: {e}");
564
565                    if !Self::sleep_or_cancel(
566                        Duration::from_millis(DEX_EVENT_STREAM_RETRY_DELAY_MS),
567                        &cancellation_token,
568                    )
569                    .await
570                    {
571                        break;
572                    }
573                    continue;
574                }
575            };
576
577            let mut received_response = false;
578
579            loop {
580                tokio::select! {
581                    () = cancellation_token.cancelled() => {
582                        log::debug!("DEX event stream task for {dex} received cancellation signal");
583                        break;
584                    }
585                    response = rx.recv() => {
586                        let Some(response) = response else {
587                            break;
588                        };
589
590                        let response = match response {
591                            Ok(resp) => resp,
592                            Err(e) => {
593                                if received_response {
594                                    log::debug!("DEX event stream drained for {dex}: {e}");
595                                } else {
596                                    log::warn!("Failed to receive DEX event stream response for {dex}: {e}");
597                                }
598                                break;
599                            }
600                        };
601
602                        received_response = true;
603                        next_from_block.fetch_max(response.next_block, Ordering::Relaxed);
604
605                        for log in response.data.logs.into_iter().flatten() {
606                            Self::send_dex_event_log(&tx, dex_extended, &log);
607                        }
608                    }
609                }
610            }
611
612            if !Self::sleep_or_cancel(
613                Duration::from_millis(DEX_EVENT_STREAM_RETRY_DELAY_MS),
614                &cancellation_token,
615            )
616            .await
617            {
618                break;
619            }
620        }
621
622        log::debug!("Stopped task 'dex_event_stream' for {dex}");
623    }
624
625    async fn wait_for_stream_start_block(
626        client: &hypersync_client::Client,
627        from_block: u64,
628        cancellation_token: &tokio_util::sync::CancellationToken,
629    ) {
630        loop {
631            match client.get_height().await {
632                Ok(height) if height >= from_block => return,
633                Ok(_) => {}
634                Err(e) => log::warn!("Failed to get HyperSync height for DEX event stream: {e}"),
635            }
636
637            if !Self::sleep_or_cancel(
638                Duration::from_millis(BLOCK_POLLING_INTERVAL_MS),
639                cancellation_token,
640            )
641            .await
642            {
643                return;
644            }
645        }
646    }
647
648    async fn sleep_or_cancel(
649        duration: Duration,
650        cancellation_token: &tokio_util::sync::CancellationToken,
651    ) -> bool {
652        tokio::select! {
653            () = cancellation_token.cancelled() => false,
654            () = tokio::time::sleep(duration) => true,
655        }
656    }
657
658    fn send_dex_event_log(
659        tx: &tokio::sync::mpsc::UnboundedSender<BlockchainMessage>,
660        dex_extended: &DexExtended,
661        log: &Log,
662    ) {
663        let event_signature = match log.topics.first().and_then(|t| t.as_ref()) {
664            Some(log_argument) => hex::encode_prefixed(log_argument.as_ref()),
665            None => return,
666        };
667
668        let (event_name, event) = if event_signature == dex_extended.swap_created_event.as_ref() {
669            (
670                "swap",
671                dex_extended
672                    .parse_swap_event_hypersync(log)
673                    .map(BlockchainMessage::SwapEvent),
674            )
675        } else if event_signature == dex_extended.mint_created_event.as_ref() {
676            (
677                "mint",
678                dex_extended
679                    .parse_mint_event_hypersync(log)
680                    .map(BlockchainMessage::MintEvent),
681            )
682        } else if event_signature == dex_extended.burn_created_event.as_ref() {
683            (
684                "burn",
685                dex_extended
686                    .parse_burn_event_hypersync(log)
687                    .map(BlockchainMessage::BurnEvent),
688            )
689        } else if event_signature == dex_extended.collect_created_event.as_ref() {
690            (
691                "collect",
692                dex_extended
693                    .parse_collect_event_hypersync(log)
694                    .map(BlockchainMessage::CollectEvent),
695            )
696        } else if dex_extended.flash_created_event.as_deref() == Some(event_signature.as_str()) {
697            (
698                "flash",
699                dex_extended
700                    .parse_flash_event_hypersync(log)
701                    .map(BlockchainMessage::FlashEvent),
702            )
703        } else if dex_extended.fee_protocol_update_event.as_deref()
704            == Some(event_signature.as_str())
705        {
706            (
707                "fee-protocol update",
708                dex_extended
709                    .parse_fee_protocol_update_event_hypersync(log)
710                    .map(BlockchainMessage::FeeProtocolUpdateEvent),
711            )
712        } else if dex_extended.fee_protocol_collect_event.as_deref()
713            == Some(event_signature.as_str())
714        {
715            (
716                "fee-protocol collect",
717                dex_extended
718                    .parse_fee_protocol_collect_event_hypersync(log)
719                    .map(BlockchainMessage::FeeProtocolCollectEvent),
720            )
721        } else {
722            log::error!("Unknown event signature: {event_signature}");
723            return;
724        };
725
726        match event {
727            Ok(event) => {
728                if let Err(e) = tx.send(event) {
729                    log::error!("Failed to send {event_name} event: {e}");
730                }
731            }
732            Err(e) => {
733                log::error!("Failed to parse {event_name} with error '{e:?}' for event: {log:?}",);
734            }
735        }
736    }
737
738    async fn stop_dex_event_stream(&mut self, dex: DexType) -> Option<u64> {
739        let task = self.dex_event_tasks.get_mut(&dex)?;
740        let terminated = Self::stop_dex_event_task(dex, task).await;
741        let next_from_block = task.next_from_block.load(Ordering::Relaxed);
742
743        if terminated {
744            self.dex_event_tasks.remove(&dex);
745        }
746
747        Some(next_from_block)
748    }
749
750    async fn stop_dex_event_task(dex: DexType, task: &mut DexEventStreamTask) -> bool {
751        task.cancellation_token.cancel();
752
753        let outcome = finish_task(
754            &mut task.task,
755            Duration::from_secs(DISCONNECT_TIMEOUT_SECS),
756            Duration::from_secs(DISCONNECT_TIMEOUT_SECS),
757        )
758        .await;
759
760        match outcome {
761            None | Some(TaskJoinOutcome::Completed(()) | TaskJoinOutcome::Aborted) => true,
762            Some(TaskJoinOutcome::Failed(error)) => {
763                log::error!("HyperSync DEX event stream task for {dex} failed: {error}");
764                true
765            }
766            Some(TaskJoinOutcome::Incomplete) => {
767                log::error!("HyperSync DEX event stream task for {dex} did not stop after abort");
768                false
769            }
770        }
771    }
772}
773
774impl Drop for HyperSyncClient {
775    fn drop(&mut self) {
776        self.cancellation_token.cancel();
777        if let Some(token) = self.blocks_cancellation_token.as_ref() {
778            token.cancel();
779        }
780        self.blocks_task.abort();
781
782        for task in self.dex_event_tasks.values_mut() {
783            task.cancellation_token.cancel();
784            task.task.abort();
785        }
786    }
787}
788
789#[derive(Debug, Clone, PartialEq, Eq)]
790struct DexEventStreamFilter {
791    contract_addresses: Vec<Address>,
792    event_signatures: Vec<String>,
793}
794
795impl DexEventStreamFilter {
796    fn new(mut contract_addresses: Vec<Address>, mut event_signatures: Vec<String>) -> Self {
797        contract_addresses.sort_unstable();
798        contract_addresses.dedup();
799        event_signatures.sort_unstable();
800        event_signatures.dedup();
801        Self {
802            contract_addresses,
803            event_signatures,
804        }
805    }
806
807    fn is_empty(&self) -> bool {
808        self.contract_addresses.is_empty() || self.event_signatures.is_empty()
809    }
810}
811
812#[derive(Debug)]
813struct DexEventStreamTask {
814    filter: DexEventStreamFilter,
815    next_from_block: Arc<AtomicU64>,
816    cancellation_token: tokio_util::sync::CancellationToken,
817    task: TaskSlot<()>,
818}
819
820/// Maps one HyperSync response into stream items, surfacing blocks ahead of the logs from the
821/// same response so callers can cache them before converting events from those blocks.
822///
823/// Blocks that fail to transform are logged and skipped without dropping the response's logs.
824fn pool_events_from_response(
825    chain: Blockchain,
826    blocks: Vec<Vec<hypersync_client::simple_types::Block>>,
827    logs: Vec<Vec<Log>>,
828) -> Vec<PoolEventStreamItem> {
829    let mut items = Vec::new();
830
831    for block in blocks.into_iter().flatten() {
832        match transform_hypersync_block(chain, block) {
833            Ok(block) => items.push(PoolEventStreamItem::Block(block)),
834            Err(e) => log::error!("Failed to transform block for timestamp: {e}"),
835        }
836    }
837
838    items.extend(logs.into_iter().flatten().map(PoolEventStreamItem::Log));
839    items
840}
841
842#[cfg(test)]
843mod tests {
844    use std::{
845        str::FromStr,
846        sync::{
847            Arc,
848            atomic::{AtomicU64, Ordering},
849        },
850    };
851
852    use hypersync_client::{
853        format::{Address as HypersyncAddress, Hash, Quantity},
854        simple_types::{Block as HypersyncBlock, Log},
855    };
856    #[cfg(not(madsim))]
857    use nautilus_common::{clients::DataClient, live::runner::replace_data_event_sender};
858    use nautilus_core::{UnixNanos, datetime::NANOSECONDS_IN_SECOND};
859    use nautilus_model::defi::Chain;
860    #[cfg(not(madsim))]
861    use nautilus_model::identifiers::ClientId;
862    use rstest::rstest;
863
864    use super::*;
865    #[cfg(not(madsim))]
866    use crate::{config::BlockchainDataClientConfig, data::client::BlockchainDataClient};
867
868    #[cfg(not(madsim))]
869    const TOKEN_TEST_ENV: &str = "BLOCKCHAIN_HYPERSYNC_TOKEN_TEST";
870
871    #[cfg(not(madsim))]
872    #[rstest]
873    #[case::missing(None, "missing")]
874    #[case::empty(Some(""), "empty")]
875    #[case::malformed(Some("not-a-uuid"), "malformed")]
876    fn test_constructor_rejects_invalid_token(#[case] token: Option<&str>, #[case] expected: &str) {
877        let module = module_path!().split_once("::").unwrap().1;
878        let child_test = format!("{module}::hypersync_constructor_token_child");
879        let executable = std::env::current_exe().unwrap();
880        let mut command = std::process::Command::new(executable);
881        command
882            .args(["--exact", &child_test, "--ignored"])
883            .env(TOKEN_TEST_ENV, expected)
884            .env_remove(API_TOKEN_ENV);
885
886        if let Some(token) = token {
887            command.env(API_TOKEN_ENV, token);
888        }
889
890        let output = command.output().unwrap();
891        let stdout = String::from_utf8(output.stdout).unwrap();
892
893        assert_eq!(output.status.code(), Some(0), "{stdout}");
894        assert!(
895            stdout.contains(&format!("test {child_test} ... ok")),
896            "{stdout}"
897        );
898    }
899
900    #[cfg(not(madsim))]
901    #[ignore = "runs only in the isolated HyperSync token test process"]
902    #[tokio::test]
903    async fn hypersync_constructor_token_child() {
904        let Ok(case) = std::env::var(TOKEN_TEST_ENV) else {
905            return;
906        };
907
908        let expected = match case.as_str() {
909            "missing" => ConfigError::missing_field(API_TOKEN_ENV),
910            "empty" => ConfigError::empty_field(API_TOKEN_ENV),
911            "malformed" => ConfigError::invalid_format(API_TOKEN_ENV, "a UUID"),
912            _ => panic!("unexpected token test case: {case}"),
913        };
914
915        let mut chain = Chain::new(Blockchain::Arbitrum, 42_161);
916        chain.hypersync_url = Some("https://42161.hypersync.xyz".to_string());
917        let chain = Arc::new(chain);
918        let error = HyperSyncClient::new(
919            Arc::clone(&chain),
920            None,
921            tokio_util::sync::CancellationToken::new(),
922        )
923        .unwrap_err();
924        let config = BlockchainDataClientConfig::builder()
925            .chain(chain)
926            .http_rpc_url("http://127.0.0.1:1".into())
927            .build();
928        let mut client = BlockchainDataClient::new(ClientId::from("BLOCKCHAIN"), config).unwrap();
929        let (sender, _receiver) = tokio::sync::mpsc::unbounded_channel();
930        replace_data_event_sender(sender);
931
932        assert_eq!(error.downcast_ref::<ConfigError>(), Some(&expected));
933
934        for _ in 0..2 {
935            let error = client.connect().await.unwrap_err();
936            assert_eq!(error.downcast_ref::<ConfigError>(), Some(&expected));
937        }
938    }
939
940    fn synthetic_block(number: u64, timestamp_secs: u64) -> HypersyncBlock {
941        HypersyncBlock {
942            number: Some(number),
943            hash: Some(
944                Hash::from_str(
945                    "0x0000000000000000000000000000000000000000000000000000000000000001",
946                )
947                .unwrap(),
948            ),
949            parent_hash: Some(
950                Hash::from_str(
951                    "0x0000000000000000000000000000000000000000000000000000000000000000",
952                )
953                .unwrap(),
954            ),
955            miner: Some(
956                HypersyncAddress::from_str("0x0000000000000000000000000000000000000001").unwrap(),
957            ),
958            gas_limit: Some(Quantity::from(21_000u64)),
959            gas_used: Some(Quantity::from(21_000u64)),
960            timestamp: Some(Quantity::from(timestamp_secs)),
961            ..Default::default()
962        }
963    }
964
965    #[rstest]
966    fn test_constructor_rejects_missing_endpoint_before_loading_token() {
967        let chain = Arc::new(Chain::new(Blockchain::Saakuru, 7_225_878));
968        let error = HyperSyncClient::new(chain, None, tokio_util::sync::CancellationToken::new())
969            .unwrap_err();
970
971        assert_eq!(
972            error.downcast_ref::<ConfigError>(),
973            Some(&ConfigError::missing_field("chain.hypersync_url")),
974        );
975    }
976
977    #[rstest]
978    fn test_constructor_rejects_remote_cleartext_before_loading_token() {
979        let mut chain = Chain::new(Blockchain::Arbitrum, 42_161);
980        chain.hypersync_url = Some("http://localhost:8080".to_string());
981
982        let error = HyperSyncClient::new(
983            Arc::new(chain),
984            None,
985            tokio_util::sync::CancellationToken::new(),
986        )
987        .unwrap_err();
988
989        assert_eq!(
990            error.downcast_ref::<ConfigError>(),
991            Some(&ConfigError::invalid_value(
992                "chain.hypersync_url",
993                "HyperSync endpoint must use HTTPS unless its host is a canonical loopback IP literal",
994            )),
995        );
996    }
997
998    #[rstest]
999    fn pool_events_yields_blocks_before_logs() {
1000        let items = pool_events_from_response(
1001            Blockchain::Ethereum,
1002            vec![vec![synthetic_block(12, 100)]],
1003            vec![vec![Log::default(), Log::default()]],
1004        );
1005
1006        assert_eq!(items.len(), 3);
1007        match &items[0] {
1008            PoolEventStreamItem::Block(block) => {
1009                assert_eq!(block.number, 12);
1010                assert_eq!(block.timestamp, UnixNanos::new(100 * NANOSECONDS_IN_SECOND));
1011            }
1012            other => panic!("expected Block first, was {other:?}"),
1013        }
1014        assert!(matches!(items[1], PoolEventStreamItem::Log(_)));
1015        assert!(matches!(items[2], PoolEventStreamItem::Log(_)));
1016    }
1017
1018    #[rstest]
1019    fn pool_events_skips_unparsable_block_but_keeps_logs() {
1020        // A block missing required fields (gas, hash, ...) fails transform and is skipped, but
1021        // the response's logs must still be yielded.
1022        let bad_block = HypersyncBlock {
1023            number: Some(7),
1024            ..Default::default()
1025        };
1026        let items = pool_events_from_response(
1027            Blockchain::Ethereum,
1028            vec![vec![bad_block]],
1029            vec![vec![Log::default()]],
1030        );
1031
1032        assert_eq!(items.len(), 1);
1033        assert!(matches!(items[0], PoolEventStreamItem::Log(_)));
1034    }
1035
1036    #[rstest]
1037    fn construct_block_query_converts_to_block_to_hypersync_exclusive_bound() {
1038        let query = HyperSyncClient::construct_block_query(10, Some(12));
1039
1040        assert_eq!(query.from_block, 10);
1041        assert_eq!(query.to_block, Some(13));
1042    }
1043
1044    #[rstest]
1045    fn construct_contract_events_query_converts_to_block_to_hypersync_exclusive_bound() {
1046        let address = Address::from_str("0x0000000000000000000000000000000000000001").unwrap();
1047        let query = HyperSyncClient::construct_contract_events_query(
1048            10,
1049            Some(12),
1050            &[address],
1051            &["0xc42079f94a6350d7e6235f29174924f928cc2ac818eb64fed8004e115fbcca67"],
1052        );
1053
1054        assert_eq!(query.from_block, 10);
1055        assert_eq!(query.to_block, Some(13));
1056    }
1057
1058    #[rstest]
1059    fn construct_contract_events_query_single_block_uses_next_block_as_exclusive_bound() {
1060        let address = Address::from_str("0x0000000000000000000000000000000000000001").unwrap();
1061        let query = HyperSyncClient::construct_contract_events_query(
1062            10,
1063            Some(10),
1064            &[address],
1065            &["0xc42079f94a6350d7e6235f29174924f928cc2ac818eb64fed8004e115fbcca67"],
1066        );
1067
1068        assert_eq!(query.from_block, 10);
1069        assert_eq!(query.to_block, Some(11));
1070    }
1071
1072    #[rstest]
1073    fn construct_contract_events_query_open_upper_bound_stays_open() {
1074        let address = Address::from_str("0x0000000000000000000000000000000000000001").unwrap();
1075        let query = HyperSyncClient::construct_contract_events_query(
1076            10,
1077            None,
1078            &[address],
1079            &["0xc42079f94a6350d7e6235f29174924f928cc2ac818eb64fed8004e115fbcca67"],
1080        );
1081
1082        assert_eq!(query.from_block, 10);
1083        assert_eq!(query.to_block, None);
1084    }
1085
1086    #[rstest]
1087    fn dex_event_stream_filter_sorts_and_deduplicates_inputs() {
1088        let address1 = Address::from_str("0x0000000000000000000000000000000000000001").unwrap();
1089        let address2 = Address::from_str("0x0000000000000000000000000000000000000002").unwrap();
1090        let filter = DexEventStreamFilter::new(
1091            vec![address2, address1, address2],
1092            vec![
1093                "0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb".to_string(),
1094                "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa".to_string(),
1095                "0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb".to_string(),
1096            ],
1097        );
1098
1099        assert_eq!(filter.contract_addresses, vec![address1, address2]);
1100        assert_eq!(
1101            filter.event_signatures,
1102            vec![
1103                "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa".to_string(),
1104                "0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb".to_string(),
1105            ]
1106        );
1107    }
1108
1109    #[rstest]
1110    fn dex_event_stream_filter_requires_addresses_and_signatures() {
1111        let address = Address::from_str("0x0000000000000000000000000000000000000001").unwrap();
1112
1113        assert!(DexEventStreamFilter::new(vec![], vec!["0x01".to_string()]).is_empty());
1114        assert!(DexEventStreamFilter::new(vec![address], vec![]).is_empty());
1115        assert!(!DexEventStreamFilter::new(vec![address], vec!["0x01".to_string()]).is_empty());
1116    }
1117
1118    #[tokio::test]
1119    async fn stop_dex_event_task_cancels_and_awaits_task() {
1120        let cancellation_token = tokio_util::sync::CancellationToken::new();
1121        let task_token = cancellation_token.clone();
1122        let next_from_block = Arc::new(AtomicU64::new(42));
1123        let task_next_from_block = Arc::clone(&next_from_block);
1124
1125        let task = tokio::spawn(async move {
1126            task_token.cancelled().await;
1127            task_next_from_block.store(99, Ordering::Relaxed);
1128        });
1129        let stream_task = DexEventStreamTask {
1130            filter: DexEventStreamFilter::new(vec![], vec![]),
1131            next_from_block,
1132            cancellation_token,
1133            task: TaskSlot::from_handle(task),
1134        };
1135        let mut stream_task = stream_task;
1136
1137        let terminated =
1138            HyperSyncClient::stop_dex_event_task(DexType::UniswapV3, &mut stream_task).await;
1139
1140        assert!(terminated);
1141        assert_eq!(stream_task.next_from_block.load(Ordering::Relaxed), 99);
1142        assert!(stream_task.task.is_none());
1143    }
1144}