Skip to main content

nautilus_blockchain/data/
core.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16use std::{cmp::max, collections::BTreeMap, sync::Arc};
17
18use ahash::AHashMap;
19use alloy::primitives::B256;
20use anyhow::Context;
21use futures_util::StreamExt;
22use nautilus_common::{live::sender::EventSender, messages::DataEvent};
23use nautilus_core::{
24    UnixNanos, hex,
25    string::{formatting::Separable, secret::REDACTED},
26};
27use nautilus_live::SocketControl;
28use nautilus_model::defi::{
29    Block, Blockchain, DexType, Pool, PoolIdentifier, PoolLiquidityUpdate, PoolProfiler, PoolSwap,
30    SharedChain, SharedDex, SharedPool,
31    data::{
32        DefiData, DexPoolData, PoolFeeCollect, PoolFeeProtocolCollect, PoolFeeProtocolUpdate,
33        PoolFlash,
34        block::{BLOCK_SCOPED_SNAPSHOT_INDEX, BlockPosition},
35    },
36    pool_analysis::{compare::compare_pool_profiler_detailed, snapshot::PoolSnapshot},
37    reporting::{BlockchainSyncReportItems, BlockchainSyncReporter},
38    tick_map::tick::PoolTick,
39};
40use nautilus_network::websocket::TransportBackend;
41
42use crate::{
43    cache::{BlockchainCache, PoolEventSyncState},
44    config::BlockchainDataClientConfig,
45    contracts::{
46        erc20::Erc20Contract,
47        uniswap_v3_pool::{FeeProtocolEncoding, UniswapV3PoolContract},
48    },
49    data::{block_hash_matches, subscription::DefiDataSubscriptionManager},
50    events::{
51        burn::BurnEvent, collect::CollectEvent, fee_protocol_collect::FeeProtocolCollectEvent,
52        fee_protocol_update::FeeProtocolUpdateEvent, flash::FlashEvent, mint::MintEvent,
53        swap::SwapEvent,
54    },
55    exchanges::{extended::DexExtended, get_dex_extended},
56    hypersync::{
57        client::{HyperSyncClient, PoolEventStreamItem},
58        log::{extract_block_number, extract_event_signature_bytes},
59    },
60    rpc::{
61        BlockchainRpcClient, BlockchainRpcClientAny,
62        chains::{
63            arbitrum::ArbitrumRpcClient, base::BaseRpcClient, bsc::BscRpcClient,
64            ethereum::EthereumRpcClient, polygon::PolygonRpcClient,
65        },
66        http::BlockchainHttpRpcClient,
67        types::BlockchainMessage,
68    },
69    services::PoolDiscoveryService,
70};
71
72const BLOCKS_PROCESS_IN_SYNC_REPORT: u64 = 50_000;
73const POOL_EVENT_BLOCK_BATCH_SIZE: usize = 20_000;
74const POOL_EVENT_SYNC_VERSION_LEGACY: u32 = 0;
75const POOL_EVENT_SYNC_VERSION_PROTOCOL_FEE: u32 = 1;
76// Give each added family a new introduction version, then advance this current version
77const POOL_EVENT_SYNC_VERSION: u32 = POOL_EVENT_SYNC_VERSION_PROTOCOL_FEE;
78// Block-scoped RPC snapshots include the whole block, so same-block replay must skip every log.
79
80#[derive(Debug, Clone, PartialEq, Eq)]
81struct PoolEventFamily {
82    name: &'static str,
83    signature: String,
84    introduced_version: u32,
85}
86
87#[derive(Debug, Clone, PartialEq, Eq)]
88struct PoolEventSyncRange {
89    from_block: u64,
90    families: Vec<PoolEventFamily>,
91}
92
93/// Core blockchain data client responsible for fetching, processing, and caching blockchain data.
94///
95/// This struct encapsulates the core functionality for interacting with blockchain networks,
96/// including syncing historical data, processing real-time events, and managing cached entities.
97#[derive(Debug)]
98pub struct BlockchainDataClientCore {
99    /// The blockchain being targeted by this client instance.
100    pub chain: SharedChain,
101    /// The configuration for the data client.
102    pub config: BlockchainDataClientConfig,
103    /// Local cache for blockchain entities.
104    pub cache: BlockchainCache,
105    /// Interface for interacting with ERC20 token contracts.
106    tokens: Erc20Contract,
107    /// Interface for interacting with UniswapV3 pool contracts.
108    univ3_pool: UniswapV3PoolContract,
109    /// Client for the HyperSync data indexing service.
110    pub hypersync_client: HyperSyncClient,
111    /// Optional WebSocket RPC client for direct blockchain node communication.
112    pub rpc_client: Option<BlockchainRpcClientAny>,
113    /// Manages subscriptions for various DEX events (swaps, mints, burns).
114    pub subscription_manager: DefiDataSubscriptionManager,
115    /// Channel sender for data events.
116    data_tx: Option<EventSender<DataEvent>>,
117    /// Cancellation token for graceful shutdown of long-running operations.
118    cancellation_token: tokio_util::sync::CancellationToken,
119}
120
121/// Outcome of validating a pool snapshot against on-chain state.
122#[derive(Debug, Clone, Copy, PartialEq, Eq)]
123pub enum SnapshotValidation {
124    /// Hydrated from chain and matched the profiler state.
125    OnChain,
126    /// Emitted from deterministic replay and not checked against chain (the RPC could not serve the
127    /// block, or validation was skipped). Usable as a replay start point.
128    Replay,
129    /// Hydrated from chain and did not match the profiler state. Not usable as a replay start point.
130    Invalid,
131}
132
133impl SnapshotValidation {
134    /// Returns `true` if the snapshot is usable as a replay start point.
135    #[must_use]
136    pub const fn is_usable(self) -> bool {
137        !matches!(self, Self::Invalid)
138    }
139
140    /// Returns the database/JSON token for this state.
141    #[must_use]
142    pub const fn as_str(self) -> &'static str {
143        match self {
144            Self::OnChain => "on_chain",
145            Self::Replay => "replay",
146            Self::Invalid => "invalid",
147        }
148    }
149
150    /// Parses the database/JSON token into a [`SnapshotValidation`].
151    ///
152    /// Returns `None` for an unrecognized token.
153    #[must_use]
154    pub fn from_db_token(token: &str) -> Option<Self> {
155        match token {
156            "on_chain" => Some(Self::OnChain),
157            "replay" => Some(Self::Replay),
158            "invalid" => Some(Self::Invalid),
159            _ => None,
160        }
161    }
162}
163
164impl BlockchainDataClientCore {
165    /// Creates a new instance of [`BlockchainDataClientCore`].
166    ///
167    /// # Errors
168    ///
169    /// Returns an error if DEX selections contain duplicates or RPC or HyperSync initialization fails.
170    pub fn new(
171        config: BlockchainDataClientConfig,
172        hypersync_tx: Option<tokio::sync::mpsc::UnboundedSender<BlockchainMessage>>,
173        data_tx: Option<EventSender<DataEvent>>,
174        cancellation_token: tokio_util::sync::CancellationToken,
175    ) -> anyhow::Result<Self> {
176        config.validate()?;
177
178        let chain = Arc::clone(&config.chain);
179        let cache = BlockchainCache::new(Arc::clone(&chain));
180
181        // Log RPC endpoints being used
182        log::debug!(
183            "Initializing blockchain data client for '{}' with HTTP RPC: {}",
184            chain.name,
185            REDACTED
186        );
187
188        let rpc_client = if let (false, Some(wss_rpc_url)) = (
189            config.use_hypersync_for_live_data,
190            config.wss_rpc_url.as_ref(),
191        ) {
192            log::debug!("WebSocket RPC URL: {REDACTED}");
193            Some(Self::initialize_rpc_client(
194                chain.name,
195                wss_rpc_url.clone().into_inner(),
196                config.transport_backend,
197                config.proxy_url.clone().map(|value| value.into_inner()),
198            ))
199        } else {
200            log::debug!("Using HyperSync for live data (no WebSocket RPC)");
201            None
202        };
203        let http_rpc_client = Arc::new(BlockchainHttpRpcClient::new(
204            config.http_rpc_url.clone().into_inner(),
205            config.rpc_requests_per_second,
206            config.proxy_url.clone().map(|value| value.into_inner()),
207        )?);
208        let multicall_calls_per_rpc_request = config.multicall_calls_per_rpc_request;
209        let erc20_contract = Erc20Contract::new(
210            Arc::clone(&http_rpc_client),
211            config.pool_filters.remove_pools_with_empty_erc20fields,
212        );
213
214        let hypersync_client =
215            HyperSyncClient::new(Arc::clone(&chain), hypersync_tx, cancellation_token.clone())?;
216        Ok(Self {
217            chain,
218            config,
219            rpc_client,
220            tokens: erc20_contract,
221            univ3_pool: UniswapV3PoolContract::new(
222                http_rpc_client,
223                multicall_calls_per_rpc_request,
224            ),
225            cache,
226            hypersync_client,
227            subscription_manager: DefiDataSubscriptionManager::new(),
228            data_tx,
229            cancellation_token,
230        })
231    }
232
233    /// Configures socket state reporting and reconnect control for direct RPC data.
234    pub fn set_socket_control(&mut self, control: SocketControl) {
235        if let Some(rpc_client) = &mut self.rpc_client {
236            rpc_client.set_socket_control(control);
237        }
238    }
239
240    /// Initializes the database connection for the blockchain cache.
241    pub async fn initialize_cache_database(&mut self) {
242        if let Some(pg_connect_options) = &self.config.postgres_cache_database_config {
243            log::debug!(
244                "Initializing blockchain cache on database '{}'",
245                pg_connect_options.database
246            );
247            self.cache
248                .initialize_database(pg_connect_options.clone().into())
249                .await;
250        }
251    }
252
253    /// Creates an appropriate blockchain RPC client for the specified blockchain.
254    fn initialize_rpc_client(
255        blockchain: Blockchain,
256        wss_rpc_url: String,
257        transport_backend: TransportBackend,
258        proxy_url: Option<String>,
259    ) -> BlockchainRpcClientAny {
260        let mut client = match blockchain {
261            Blockchain::Ethereum => {
262                BlockchainRpcClientAny::Ethereum(EthereumRpcClient::new(wss_rpc_url, proxy_url))
263            }
264            Blockchain::Polygon => {
265                BlockchainRpcClientAny::Polygon(PolygonRpcClient::new(wss_rpc_url, proxy_url))
266            }
267            Blockchain::Base => {
268                BlockchainRpcClientAny::Base(BaseRpcClient::new(wss_rpc_url, proxy_url))
269            }
270            Blockchain::Arbitrum => {
271                BlockchainRpcClientAny::Arbitrum(ArbitrumRpcClient::new(wss_rpc_url, proxy_url))
272            }
273            Blockchain::Bsc => {
274                BlockchainRpcClientAny::Bsc(BscRpcClient::new(wss_rpc_url, proxy_url))
275            }
276            _ => panic!("Unsupported blockchain {blockchain} for RPC connection"),
277        };
278        client.set_transport_backend(transport_backend);
279        client
280    }
281
282    /// Establishes connections to all configured data sources and initializes the cache.
283    ///
284    /// # Errors
285    ///
286    /// Returns an error if DEX selections contain duplicates, cache initialization fails, or
287    /// connection setup fails.
288    pub async fn connect(&mut self) -> anyhow::Result<()> {
289        self.config.validate()?;
290
291        log::debug!(
292            "Connecting blockchain data client for '{}'",
293            self.chain.name
294        );
295        self.initialize_cache_database().await;
296
297        if let Some(ref mut rpc_client) = self.rpc_client {
298            rpc_client.connect().await?;
299        }
300
301        let from_block = self.determine_from_block();
302
303        log::debug!(
304            "Connecting to blockchain data source for '{}' from block {}",
305            self.chain.name,
306            from_block.separate_with_commas()
307        );
308
309        // Initialize the chain and register the Dex exchanges in the cache.
310        self.cache.initialize_chain().await;
311        // Import the cached blockchain data.
312        self.cache.connect(from_block).await?;
313        // TODO disable block syncing for now as we don't have timestamps yet configured
314        // Sync the remaining blocks which are missing.
315        // self.sync_blocks(Some(from_block), None).await?;
316        for dex in self.config.dex_ids.clone() {
317            self.register_dex_exchange(dex).await?;
318            self.sync_exchange_pools(&dex, from_block, None, false)
319                .await?;
320        }
321
322        Ok(())
323    }
324
325    /// Syncs blocks with consistency checks to ensure data integrity.
326    ///
327    /// # Errors
328    ///
329    /// Returns an error if block syncing fails or if consistency checks fail.
330    pub async fn sync_blocks_checked(
331        &mut self,
332        from_block: u64,
333        to_block: Option<u64>,
334    ) -> anyhow::Result<()> {
335        if let Some(blocks_status) = self.cache.get_cache_block_consistency_status().await {
336            // If blocks are consistent proceed with copy command.
337            if blocks_status.is_consistent() {
338                log::debug!(
339                    "Cache is consistent: no gaps detected (last continuous block: {})",
340                    blocks_status.last_continuous_block
341                );
342                let target_block = max(blocks_status.max_block + 1, from_block);
343                log::debug!(
344                    "Starting fast sync with COPY from block {}",
345                    target_block.separate_with_commas()
346                );
347                self.sync_blocks(target_block, to_block, true).await?;
348            } else {
349                let gap_size = blocks_status.max_block - blocks_status.last_continuous_block;
350                log::debug!(
351                    "Cache inconsistency detected: {} blocks missing between {} and {}",
352                    gap_size,
353                    blocks_status.last_continuous_block + 1,
354                    blocks_status.max_block
355                );
356
357                log::debug!(
358                    "Block syncing Phase 1: Filling gaps with INSERT (blocks {} to {})",
359                    blocks_status.last_continuous_block + 1,
360                    blocks_status.max_block
361                );
362                self.sync_blocks(
363                    blocks_status.last_continuous_block + 1,
364                    Some(blocks_status.max_block),
365                    false,
366                )
367                .await?;
368
369                log::debug!(
370                    "Block syncing Phase 2: Continuing with fast COPY from block {}",
371                    (blocks_status.max_block + 1).separate_with_commas()
372                );
373                self.sync_blocks(blocks_status.max_block + 1, to_block, true)
374                    .await?;
375            }
376        } else {
377            self.sync_blocks(from_block, to_block, true).await?;
378        }
379
380        Ok(())
381    }
382
383    /// Synchronizes blockchain data by fetching and caching all blocks from the starting block to the current chain head.
384    ///
385    /// # Errors
386    ///
387    /// Returns an error if block fetching, caching, or database operations fail.
388    pub async fn sync_blocks(
389        &mut self,
390        from_block: u64,
391        to_block: Option<u64>,
392        use_copy_command: bool,
393    ) -> anyhow::Result<()> {
394        const BATCH_SIZE: usize = 1000;
395
396        let to_block = if let Some(block) = to_block {
397            block
398        } else {
399            self.hypersync_client.current_block().await
400        };
401        let total_blocks = to_block.saturating_sub(from_block) + 1;
402        log::debug!(
403            "Syncing blocks from {} to {} (total: {} blocks)",
404            from_block.separate_with_commas(),
405            to_block.separate_with_commas(),
406            total_blocks.separate_with_commas()
407        );
408
409        // Enable performance settings for sync operations
410        if let Err(e) = self.cache.toggle_performance_settings(true).await {
411            log::warn!("Failed to enable performance settings: {e}");
412        }
413
414        let blocks_stream = self
415            .hypersync_client
416            .request_blocks_stream(from_block, Some(to_block))
417            .await;
418
419        tokio::pin!(blocks_stream);
420
421        let mut metrics = BlockchainSyncReporter::new(
422            BlockchainSyncReportItems::Blocks,
423            from_block,
424            total_blocks,
425            BLOCKS_PROCESS_IN_SYNC_REPORT,
426        );
427
428        let mut batch: Vec<Block> = Vec::with_capacity(BATCH_SIZE);
429
430        let cancellation_token = self.cancellation_token.clone();
431        let sync_result = tokio::select! {
432            () = cancellation_token.cancelled() => {
433                log::debug!("Block sync cancelled");
434                Err(anyhow::anyhow!("Sync cancelled"))
435            }
436            result = async {
437                while let Some(block) = blocks_stream.next().await {
438                    let block_number = block.number;
439                    if self.cache.get_block_timestamp(block_number).is_some() {
440                        continue;
441                    }
442                    batch.push(block);
443
444                    // Process batch when full or last block
445                    if batch.len() >= BATCH_SIZE || block_number >= to_block {
446                        let batch_size = batch.len();
447
448                        self.cache.add_blocks_batch(batch, use_copy_command).await?;
449                        metrics.update(batch_size);
450
451                        // Re-initialize batch vector
452                        batch = Vec::with_capacity(BATCH_SIZE);
453                    }
454
455                    // Log progress if needed
456                    if metrics.should_log_progress(block_number, to_block) {
457                        metrics.log_progress(block_number);
458                    }
459                }
460
461                // Process any remaining blocks
462                if !batch.is_empty() {
463                    let batch_size = batch.len();
464                    self.cache.add_blocks_batch(batch, use_copy_command).await?;
465                    metrics.update(batch_size);
466                }
467
468                metrics.log_final_stats();
469                Ok(())
470            } => result
471        };
472
473        sync_result?;
474
475        // Restore default safe settings after sync completion
476        if let Err(e) = self.cache.toggle_performance_settings(false).await {
477            log::warn!("Failed to restore default settings: {e}");
478        }
479
480        Ok(())
481    }
482
483    /// Synchronizes all events for a specific pool within the given block range.
484    ///
485    /// # Errors
486    ///
487    /// Returns an error if event syncing, parsing, or database operations fail.
488    pub async fn sync_pool_events(
489        &mut self,
490        dex: &DexType,
491        pool_identifier: PoolIdentifier,
492        from_block: Option<u64>,
493        to_block: Option<u64>,
494        reset: bool,
495    ) -> anyhow::Result<()> {
496        const EVENT_BATCH_SIZE: usize = 20000;
497
498        let pool: SharedPool = Arc::clone(self.get_pool(&pool_identifier)?);
499        let pool_display = pool.to_full_spec_string();
500        let from_block = from_block.unwrap_or(pool.creation_block);
501        // Extract address for blockchain queries
502        let pool_address = &pool.address;
503
504        let dex_extended = self.get_dex_extended(dex)?.clone();
505        let event_families = Self::pool_event_families(&dex_extended);
506        let sync_state = if reset {
507            PoolEventSyncState::default()
508        } else {
509            self.cache
510                .get_pool_event_sync_state(dex, &pool_identifier)
511                .await?
512        };
513
514        let to_block = match to_block {
515            Some(block) => block,
516            None => self.hypersync_client.current_block().await,
517        };
518
519        if from_block > to_block {
520            log::debug!(
521                "D {} Pool '{}' requested event range {} to {} is empty, skipping sync",
522                dex,
523                pool_display,
524                from_block.separate_with_commas(),
525                to_block.separate_with_commas(),
526            );
527            return Ok(());
528        }
529
530        let sync_ranges =
531            Self::pool_event_sync_ranges(&event_families, &sync_state, from_block, to_block);
532        let inherited_event_family_names = event_families
533            .iter()
534            .filter(|family| {
535                sync_state.version == POOL_EVENT_SYNC_VERSION_LEGACY
536                    && family.introduced_version == POOL_EVENT_SYNC_VERSION_LEGACY
537                    && !sync_state
538                        .family_blocks
539                        .iter()
540                        .any(|(name, _)| name == family.name)
541            })
542            .map(|family| family.name)
543            .collect::<Vec<_>>();
544
545        let Some(sync_range) = sync_ranges.first().cloned() else {
546            if sync_state.version < POOL_EVENT_SYNC_VERSION {
547                if let Some(last_full_sync_block) = sync_state.last_full_sync_block {
548                    self.cache
549                        .update_pool_event_sync(
550                            dex,
551                            &pool_identifier,
552                            &inherited_event_family_names,
553                            last_full_sync_block,
554                            None,
555                        )
556                        .await?;
557                }
558                self.cache
559                    .update_pool_event_sync(
560                        dex,
561                        &pool_identifier,
562                        &[],
563                        to_block,
564                        Some(POOL_EVENT_SYNC_VERSION),
565                    )
566                    .await?;
567            }
568
569            log::debug!(
570                "D {} Pool '{}' event families already synced to block {}, skipping sync",
571                dex,
572                pool_display,
573                to_block.separate_with_commas()
574            );
575            return Ok(());
576        };
577        let has_more_ranges = sync_ranges.len() > 1;
578
579        // Query table max blocks to detect last blocks to use batch insert before that, then COPY command.
580        let last_block_across_pool_events_table = self
581            .cache
582            .get_pool_event_tables_last_block(&pool_identifier)
583            .await?;
584
585        let effective_from_block = sync_range.from_block;
586        let event_family_names = sync_range
587            .families
588            .iter()
589            .map(|family| family.name)
590            .collect::<Vec<_>>();
591        let total_blocks = to_block.saturating_sub(effective_from_block) + 1;
592        log::debug!(
593            "Syncing Pool: '{}' event families {:?} from {} to {} (total: {} blocks)",
594            pool_display,
595            event_family_names,
596            effective_from_block.separate_with_commas(),
597            to_block.separate_with_commas(),
598            total_blocks.separate_with_commas(),
599        );
600
601        let mut metrics = BlockchainSyncReporter::new(
602            BlockchainSyncReportItems::PoolEvents,
603            effective_from_block,
604            total_blocks,
605            BLOCKS_PROCESS_IN_SYNC_REPORT,
606        );
607        let swap_event_signature = dex_extended.swap_created_event.as_ref();
608        let mint_event_signature = dex_extended.mint_created_event.as_ref();
609        let burn_event_signature = dex_extended.burn_created_event.as_ref();
610        let collect_event_signature = dex_extended.collect_created_event.as_ref();
611        let flash_event_signature = dex_extended.flash_created_event.as_ref();
612        let protocol_update_event_signature = dex_extended.fee_protocol_update_event.as_ref();
613        let protocol_collect_event_signature = dex_extended.fee_protocol_collect_event.as_ref();
614        let initialize_event_signature: Option<&str> =
615            dex_extended.initialize_event.as_ref().map(|s| s.as_ref());
616
617        // Pre-decode event signatures to bytes for efficient comparison
618        let swap_sig_bytes = hex::decode(
619            swap_event_signature
620                .strip_prefix("0x")
621                .unwrap_or(swap_event_signature),
622        )?;
623        let mint_sig_bytes = hex::decode(
624            mint_event_signature
625                .strip_prefix("0x")
626                .unwrap_or(mint_event_signature),
627        )?;
628        let burn_sig_bytes = hex::decode(
629            burn_event_signature
630                .strip_prefix("0x")
631                .unwrap_or(burn_event_signature),
632        )?;
633        let collect_sig_bytes = hex::decode(
634            collect_event_signature
635                .strip_prefix("0x")
636                .unwrap_or(collect_event_signature),
637        )?;
638        let flash_sig_bytes = flash_event_signature
639            .map(|s| hex::decode(s.strip_prefix("0x").unwrap_or(s)).unwrap_or_default());
640        let protocol_update_sig_bytes = protocol_update_event_signature
641            .map(|s| hex::decode(s.strip_prefix("0x").unwrap_or(s)).unwrap_or_default());
642        let protocol_collect_sig_bytes = protocol_collect_event_signature
643            .map(|s| hex::decode(s.strip_prefix("0x").unwrap_or(s)).unwrap_or_default());
644        let initialize_sig_bytes = initialize_event_signature
645            .map(|s| hex::decode(s.strip_prefix("0x").unwrap_or(s)).unwrap_or_default());
646
647        let event_signatures = sync_range
648            .families
649            .iter()
650            .map(|family| family.signature.as_str())
651            .collect();
652
653        let pool_events_stream = self
654            .hypersync_client
655            .request_contract_events_stream(
656                effective_from_block,
657                Some(to_block),
658                pool_address,
659                event_signatures,
660            )
661            .await;
662        tokio::pin!(pool_events_stream);
663
664        let mut last_block_saved = effective_from_block;
665        let mut blocks_processed = 0;
666
667        let mut block_batch: Vec<Block> = Vec::with_capacity(POOL_EVENT_BLOCK_BATCH_SIZE);
668        let mut swap_batch: Vec<PoolSwap> = Vec::with_capacity(EVENT_BATCH_SIZE);
669        let mut liquidity_batch: Vec<PoolLiquidityUpdate> = Vec::with_capacity(EVENT_BATCH_SIZE);
670        let mut collect_batch: Vec<PoolFeeCollect> = Vec::with_capacity(EVENT_BATCH_SIZE);
671        let mut protocol_update_batch: Vec<PoolFeeProtocolUpdate> =
672            Vec::with_capacity(EVENT_BATCH_SIZE);
673        let mut protocol_collect_batch: Vec<PoolFeeProtocolCollect> =
674            Vec::with_capacity(EVENT_BATCH_SIZE);
675        let mut flash_batch: Vec<PoolFlash> = Vec::with_capacity(EVENT_BATCH_SIZE);
676
677        // Track when we've moved beyond stale data and can use COPY
678        let mut beyond_stale_data = last_block_across_pool_events_table
679            .is_none_or(|tables_max| effective_from_block > tables_max);
680
681        let cancellation_token = self.cancellation_token.clone();
682        let sync_result = tokio::select! {
683            () = cancellation_token.cancelled() => {
684                log::debug!("Pool event sync cancelled");
685                Err(anyhow::anyhow!("Sync cancelled"))
686            }
687            result = async {
688                while let Some(item) = pool_events_stream.next().await {
689                    let log = match item {
690                        PoolEventStreamItem::Block(block) => {
691                            self.record_pool_event_block(block, &mut block_batch).await?;
692                            continue;
693                        }
694                        PoolEventStreamItem::Log(log) => log,
695                    };
696                    let block_number = extract_block_number(&log)?;
697                    blocks_processed += block_number - last_block_saved;
698                    last_block_saved = block_number;
699
700                    let event_sig_bytes = extract_event_signature_bytes(&log)?;
701            if event_sig_bytes == swap_sig_bytes.as_slice() {
702                let swap_event = dex_extended.parse_swap_event_hypersync(&log)?;
703                let swap = self
704                    .process_pool_swap_event(&swap_event, &pool)
705                    .with_context(|| {
706                        format!("failed to process swap event at block {}", swap_event.block_number)
707                    })?;
708                swap_batch.push(swap);
709            } else if event_sig_bytes == mint_sig_bytes.as_slice() {
710                let mint_event = dex_extended.parse_mint_event_hypersync(&log)?;
711                let liquidity_update = self
712                    .process_pool_mint_event(&mint_event, &pool, &dex_extended)
713                    .with_context(|| {
714                        format!("failed to process mint event at block {}", mint_event.block_number)
715                    })?;
716                liquidity_batch.push(liquidity_update);
717            } else if event_sig_bytes == burn_sig_bytes.as_slice() {
718                let burn_event = dex_extended.parse_burn_event_hypersync(&log)?;
719                let liquidity_update = self
720                    .process_pool_burn_event(&burn_event, &pool, &dex_extended)
721                    .with_context(|| {
722                        format!("failed to process burn event at block {}", burn_event.block_number)
723                    })?;
724                liquidity_batch.push(liquidity_update);
725            } else if event_sig_bytes == collect_sig_bytes.as_slice() {
726                let collect_event = dex_extended.parse_collect_event_hypersync(&log)?;
727                let fee_collect = self
728                    .process_pool_collect_event(&collect_event, &pool, &dex_extended)
729                    .with_context(|| {
730                        format!(
731                            "failed to process collect event at block {}",
732                            collect_event.block_number
733                        )
734                    })?;
735                collect_batch.push(fee_collect);
736            } else if initialize_sig_bytes.as_ref().is_some_and(|sig| sig.as_slice() == event_sig_bytes) {
737                let initialize_event = dex_extended.parse_initialize_event_hypersync(&log)?;
738                self.cache
739                    .update_pool_initialize_price_tick(&initialize_event)
740                    .await?;
741            } else if protocol_update_sig_bytes.as_ref().is_some_and(|sig| sig.as_slice() == event_sig_bytes) {
742                let fee_protocol_update_event = dex_extended.parse_fee_protocol_update_event_hypersync(&log)?;
743                let update = self
744                    .process_pool_fee_protocol_update_event(&fee_protocol_update_event, &pool)
745                    .with_context(|| {
746                        format!(
747                            "failed to process SetFeeProtocol event at block {}",
748                            fee_protocol_update_event.block_number
749                        )
750                    })?;
751                protocol_update_batch.push(update);
752            } else if protocol_collect_sig_bytes.as_ref().is_some_and(|sig| sig.as_slice() == event_sig_bytes) {
753                let fee_protocol_collect_event = dex_extended.parse_fee_protocol_collect_event_hypersync(&log)?;
754                let collect = self
755                    .process_pool_fee_protocol_collect_event(&fee_protocol_collect_event, &pool)
756                    .with_context(|| {
757                        format!(
758                            "failed to process CollectProtocol event at block {}",
759                            fee_protocol_collect_event.block_number
760                        )
761                    })?;
762                protocol_collect_batch.push(collect);
763            } else if flash_sig_bytes.as_ref().is_some_and(|sig| sig.as_slice() == event_sig_bytes) {
764                let parse_fn = dex_extended
765                    .parse_flash_event_hypersync_fn
766                    .context("missing flash event parser")?;
767                let flash_event = parse_fn(Arc::clone(&dex_extended.dex), &log)
768                    .context("failed to parse flash event")?;
769                let flash = self
770                    .process_pool_flash_event(&flash_event, &pool)
771                    .with_context(|| {
772                        format!("failed to process flash event at block {}", flash_event.block_number)
773                    })?;
774                flash_batch.push(flash);
775            } else {
776                let event_signature = hex::encode(event_sig_bytes);
777                anyhow::bail!("unexpected event signature {event_signature} for log {log:?}");
778            }
779
780            // Check if we've moved beyond stale data (transition point for strategy change)
781            if !beyond_stale_data
782                && last_block_across_pool_events_table
783                    .is_some_and(|table_max| block_number > table_max)
784            {
785                log::debug!(
786                    "Crossed beyond stale data at block {block_number} - flushing current batches with ON CONFLICT, then switching to COPY"
787                );
788
789                // Flush all batches with ON CONFLICT to handle any remaining duplicates
790                self.flush_event_batches(
791                    EVENT_BATCH_SIZE,
792                    &mut block_batch,
793                    &mut swap_batch,
794                    &mut liquidity_batch,
795                    &mut collect_batch,
796                    &mut protocol_update_batch,
797                    &mut protocol_collect_batch,
798                    &mut flash_batch,
799                    false,
800                    true,
801                )
802                .await?;
803
804                beyond_stale_data = true;
805                log::debug!("Switched to COPY mode - future batches will use COPY command");
806            } else {
807                // Process batches when they reach batch size
808                self.flush_event_batches(
809                    EVENT_BATCH_SIZE,
810                    &mut block_batch,
811                    &mut swap_batch,
812                    &mut liquidity_batch,
813                    &mut collect_batch,
814                    &mut protocol_update_batch,
815                    &mut protocol_collect_batch,
816                    &mut flash_batch,
817                    false, // TODO temporary dont use copy command
818                    false,
819                )
820                .await?;
821            }
822
823            metrics.update(blocks_processed as usize);
824            blocks_processed = 0;
825
826            // Log progress if needed
827            if metrics.should_log_progress(block_number, to_block) {
828                metrics.log_progress(block_number);
829                self.flush_event_batches(
830                    EVENT_BATCH_SIZE,
831                    &mut block_batch,
832                    &mut swap_batch,
833                    &mut liquidity_batch,
834                    &mut collect_batch,
835                    &mut protocol_update_batch,
836                    &mut protocol_collect_batch,
837                    &mut flash_batch,
838                    false,
839                    true,
840                )
841                .await?;
842
843                if let Some(checkpoint_block) =
844                    Self::completed_pool_event_checkpoint(block_number, effective_from_block)
845                {
846                    self.cache
847                        .update_pool_event_sync(
848                            dex,
849                            &pool_identifier,
850                            &event_family_names,
851                            checkpoint_block,
852                            None,
853                        )
854                        .await?;
855                }
856            }
857        }
858
859        self.flush_event_batches(
860            EVENT_BATCH_SIZE,
861            &mut block_batch,
862            &mut swap_batch,
863            &mut liquidity_batch,
864            &mut collect_batch,
865            &mut protocol_update_batch,
866            &mut protocol_collect_batch,
867            &mut flash_batch,
868            false,
869            true,
870        )
871        .await?;
872
873        metrics.log_final_stats();
874        self.cache
875            .update_pool_event_sync(
876                dex,
877                &pool_identifier,
878                &event_family_names,
879                to_block,
880                None,
881            )
882            .await?;
883
884        log::debug!(
885            "Successfully synced Dex '{}' Pool '{}' event families {:?} up to block {}",
886            dex,
887            pool_display,
888            event_family_names,
889            to_block.separate_with_commas()
890        );
891                Ok(())
892            } => result
893        };
894
895        sync_result?;
896
897        if has_more_ranges {
898            return Box::pin(self.sync_pool_events(
899                dex,
900                pool_identifier,
901                Some(from_block),
902                Some(to_block),
903                false,
904            ))
905            .await;
906        }
907
908        if let Some(last_full_sync_block) = sync_state.last_full_sync_block {
909            self.cache
910                .update_pool_event_sync(
911                    dex,
912                    &pool_identifier,
913                    &inherited_event_family_names,
914                    last_full_sync_block,
915                    None,
916                )
917                .await?;
918        }
919        self.cache
920            .update_pool_event_sync(
921                dex,
922                &pool_identifier,
923                &[],
924                to_block,
925                Some(POOL_EVENT_SYNC_VERSION),
926            )
927            .await?;
928
929        Ok(())
930    }
931
932    fn pool_event_families(dex: &DexExtended) -> Vec<PoolEventFamily> {
933        let mut families = vec![
934            PoolEventFamily {
935                name: "swap",
936                signature: dex.swap_created_event.to_string(),
937                introduced_version: POOL_EVENT_SYNC_VERSION_LEGACY,
938            },
939            PoolEventFamily {
940                name: "mint",
941                signature: dex.mint_created_event.to_string(),
942                introduced_version: POOL_EVENT_SYNC_VERSION_LEGACY,
943            },
944            PoolEventFamily {
945                name: "burn",
946                signature: dex.burn_created_event.to_string(),
947                introduced_version: POOL_EVENT_SYNC_VERSION_LEGACY,
948            },
949            PoolEventFamily {
950                name: "collect",
951                signature: dex.collect_created_event.to_string(),
952                introduced_version: POOL_EVENT_SYNC_VERSION_LEGACY,
953            },
954        ];
955
956        if let Some(signature) = &dex.initialize_event {
957            families.push(PoolEventFamily {
958                name: "initialize",
959                signature: signature.to_string(),
960                introduced_version: POOL_EVENT_SYNC_VERSION_LEGACY,
961            });
962        }
963
964        if let Some(signature) = &dex.flash_created_event {
965            families.push(PoolEventFamily {
966                name: "flash",
967                signature: signature.to_string(),
968                introduced_version: POOL_EVENT_SYNC_VERSION_LEGACY,
969            });
970        }
971
972        if let Some(signature) = &dex.fee_protocol_update_event {
973            families.push(PoolEventFamily {
974                name: "fee_protocol_update",
975                signature: signature.to_string(),
976                introduced_version: POOL_EVENT_SYNC_VERSION_PROTOCOL_FEE,
977            });
978        }
979
980        if let Some(signature) = &dex.fee_protocol_collect_event {
981            families.push(PoolEventFamily {
982                name: "fee_protocol_collect",
983                signature: signature.to_string(),
984                introduced_version: POOL_EVENT_SYNC_VERSION_PROTOCOL_FEE,
985            });
986        }
987
988        families
989    }
990
991    fn pool_event_sync_ranges(
992        families: &[PoolEventFamily],
993        state: &PoolEventSyncState,
994        from_block: u64,
995        to_block: u64,
996    ) -> Vec<PoolEventSyncRange> {
997        let mut grouped = BTreeMap::<u64, Vec<PoolEventFamily>>::new();
998
999        for family in families {
1000            let explicit_checkpoint = state
1001                .family_blocks
1002                .iter()
1003                .find_map(|(name, block)| (name == family.name).then_some(*block));
1004            let legacy_checkpoint = (state.version == POOL_EVENT_SYNC_VERSION_LEGACY
1005                && family.introduced_version == POOL_EVENT_SYNC_VERSION_LEGACY)
1006                .then_some(state.last_full_sync_block)
1007                .flatten();
1008            let family_from_block = explicit_checkpoint
1009                .or(legacy_checkpoint)
1010                .map_or(from_block, |block| max(from_block, block.saturating_add(1)));
1011
1012            if family_from_block <= to_block {
1013                grouped
1014                    .entry(family_from_block)
1015                    .or_default()
1016                    .push(family.clone());
1017            }
1018        }
1019
1020        grouped
1021            .into_iter()
1022            .map(|(from_block, families)| PoolEventSyncRange {
1023                from_block,
1024                families,
1025            })
1026            .collect()
1027    }
1028
1029    #[expect(clippy::too_many_arguments)]
1030    async fn flush_event_batches(
1031        &mut self,
1032        event_batch_size: usize,
1033        block_batch: &mut Vec<Block>,
1034        swap_batch: &mut Vec<PoolSwap>,
1035        liquidity_batch: &mut Vec<PoolLiquidityUpdate>,
1036        collect_batch: &mut Vec<PoolFeeCollect>,
1037        protocol_update_batch: &mut Vec<PoolFeeProtocolUpdate>,
1038        protocol_collect_batch: &mut Vec<PoolFeeProtocolCollect>,
1039        flash_batch: &mut Vec<PoolFlash>,
1040        use_copy_command: bool,
1041        force_flush_all: bool,
1042    ) -> anyhow::Result<()> {
1043        let should_flush_swaps =
1044            (force_flush_all || swap_batch.len() >= event_batch_size) && !swap_batch.is_empty();
1045        let should_flush_liquidity = (force_flush_all || liquidity_batch.len() >= event_batch_size)
1046            && !liquidity_batch.is_empty();
1047        let should_flush_collects = (force_flush_all || collect_batch.len() >= event_batch_size)
1048            && !collect_batch.is_empty();
1049        let should_flush_protocol_update = (force_flush_all
1050            || protocol_update_batch.len() >= event_batch_size)
1051            && !protocol_update_batch.is_empty();
1052        let should_flush_protocol_collect = (force_flush_all
1053            || protocol_collect_batch.len() >= event_batch_size)
1054            && !protocol_collect_batch.is_empty();
1055        let should_flush_flash =
1056            (force_flush_all || flash_batch.len() >= event_batch_size) && !flash_batch.is_empty();
1057
1058        if force_flush_all
1059            || should_flush_swaps
1060            || should_flush_liquidity
1061            || should_flush_collects
1062            || should_flush_protocol_update
1063            || should_flush_protocol_collect
1064            || should_flush_flash
1065        {
1066            self.flush_pool_event_blocks(block_batch).await?;
1067        }
1068
1069        if should_flush_swaps {
1070            self.cache
1071                .add_pool_swaps_batch(swap_batch, use_copy_command)
1072                .await?;
1073            swap_batch.clear();
1074        }
1075
1076        if should_flush_liquidity {
1077            self.cache
1078                .add_pool_liquidity_updates_batch(liquidity_batch, use_copy_command)
1079                .await?;
1080            liquidity_batch.clear();
1081        }
1082
1083        if should_flush_collects {
1084            self.cache
1085                .add_pool_fee_collects_batch(collect_batch, use_copy_command)
1086                .await?;
1087            collect_batch.clear();
1088        }
1089
1090        if should_flush_protocol_update {
1091            self.cache
1092                .add_pool_fee_protocol_updates_batch(protocol_update_batch)
1093                .await?;
1094            protocol_update_batch.clear();
1095        }
1096
1097        if should_flush_protocol_collect {
1098            self.cache
1099                .add_pool_fee_protocol_collect_batch(protocol_collect_batch)
1100                .await?;
1101            protocol_collect_batch.clear();
1102        }
1103
1104        if should_flush_flash {
1105            self.cache.add_pool_flash_batch(flash_batch).await?;
1106            flash_batch.clear();
1107        }
1108        Ok(())
1109    }
1110
1111    async fn record_pool_event_block(
1112        &mut self,
1113        block: Block,
1114        block_batch: &mut Vec<Block>,
1115    ) -> anyhow::Result<()> {
1116        self.cache.cache_block_metadata(&block)?;
1117        block_batch.push(block);
1118        if block_batch.len() >= POOL_EVENT_BLOCK_BATCH_SIZE {
1119            self.flush_pool_event_blocks(block_batch).await?;
1120        }
1121        Ok(())
1122    }
1123
1124    async fn flush_pool_event_blocks(
1125        &mut self,
1126        block_batch: &mut Vec<Block>,
1127    ) -> anyhow::Result<()> {
1128        if block_batch.is_empty() {
1129            return Ok(());
1130        }
1131
1132        self.cache
1133            .add_pool_event_blocks_batch(std::mem::take(block_batch))
1134            .await
1135    }
1136
1137    fn completed_pool_event_checkpoint(
1138        block_number: u64,
1139        effective_from_block: u64,
1140    ) -> Option<u64> {
1141        let checkpoint_block = block_number.checked_sub(1)?;
1142        (checkpoint_block >= effective_from_block).then_some(checkpoint_block)
1143    }
1144
1145    /// Processes a swap event and converts it to a pool swap.
1146    ///
1147    /// Trade-info computation can fail for degenerate MIN/MAX-tick swaps on near-zero-liquidity
1148    /// pools, whose spot price overflows the price representation. Such failures are non-fatal: the
1149    /// swap is kept with empty trade-info so an otherwise-valid event does not abort the pool sync.
1150    ///
1151    /// # Errors
1152    ///
1153    /// Returns an error if the swap event's block timestamp or cached block hash
1154    /// is missing from the cache, or if its observed block hash does not match
1155    /// the cached hash.
1156    pub fn process_pool_swap_event(
1157        &self,
1158        swap_event: &SwapEvent,
1159        pool: &SharedPool,
1160    ) -> anyhow::Result<PoolSwap> {
1161        let timestamp = self
1162            .cache
1163            .get_block_timestamp(swap_event.block_number)
1164            .copied()
1165            .context("missing block timestamp for swap event")?;
1166        let mut swap = swap_event.to_pool_swap(
1167            Arc::clone(&self.chain),
1168            pool.instrument_id,
1169            pool.pool_identifier,
1170            timestamp,
1171        );
1172        swap.block_hash = Some(self.observed_block_hash(
1173            swap_event.block_number,
1174            swap_event.block_hash,
1175            "swap",
1176        )?);
1177
1178        // Keep the swap and leave price metadata empty rather than aborting the pool sync
1179        if let Err(e) = swap.calculate_trade_info(&pool.token0, &pool.token1, None) {
1180            log::debug!(
1181                "Skipping trade info for swap at block {} on pool {}: {e}",
1182                swap_event.block_number,
1183                pool.instrument_id,
1184            );
1185        }
1186
1187        Ok(swap)
1188    }
1189
1190    /// Processes a mint event (liquidity addition) and converts it to a `PoolLiquidityUpdate`.
1191    ///
1192    /// # Errors
1193    ///
1194    /// Returns an error if mint event processing fails or if the liquidity update creation fails.
1195    pub fn process_pool_mint_event(
1196        &self,
1197        mint_event: &MintEvent,
1198        pool: &SharedPool,
1199        dex_extended: &DexExtended,
1200    ) -> anyhow::Result<PoolLiquidityUpdate> {
1201        let timestamp = self
1202            .cache
1203            .get_block_timestamp(mint_event.block_number)
1204            .copied()
1205            .context("missing block timestamp for mint event")?;
1206
1207        let mut liquidity_update = mint_event.to_pool_liquidity_update(
1208            Arc::clone(&self.chain),
1209            Arc::clone(&dex_extended.dex),
1210            pool.instrument_id,
1211            timestamp,
1212        );
1213        liquidity_update.block_hash = Some(self.observed_block_hash(
1214            mint_event.block_number,
1215            mint_event.block_hash,
1216            "mint",
1217        )?);
1218
1219        // self.cache.add_liquidity_update(&liquidity_update).await?;
1220
1221        Ok(liquidity_update)
1222    }
1223
1224    /// Processes a burn event (liquidity removal) and converts it to a `PoolLiquidityUpdate`.
1225    /// Processes a pool burn event and converts it to a liquidity update.
1226    ///
1227    /// # Errors
1228    ///
1229    /// Returns an error if the burn event processing fails or if the liquidity update creation fails.
1230    pub fn process_pool_burn_event(
1231        &self,
1232        burn_event: &BurnEvent,
1233        pool: &SharedPool,
1234        dex_extended: &DexExtended,
1235    ) -> anyhow::Result<PoolLiquidityUpdate> {
1236        let timestamp = self
1237            .cache
1238            .get_block_timestamp(burn_event.block_number)
1239            .copied()
1240            .context("missing block timestamp for burn event")?;
1241
1242        let mut liquidity_update = burn_event.to_pool_liquidity_update(
1243            Arc::clone(&self.chain),
1244            Arc::clone(&dex_extended.dex),
1245            pool.instrument_id,
1246            pool.pool_identifier,
1247            timestamp,
1248        );
1249        liquidity_update.block_hash = Some(self.observed_block_hash(
1250            burn_event.block_number,
1251            burn_event.block_hash,
1252            "burn",
1253        )?);
1254
1255        // self.cache.add_liquidity_update(&liquidity_update).await?;
1256
1257        Ok(liquidity_update)
1258    }
1259
1260    /// Processes a pool collect event and converts it to a fee collection.
1261    ///
1262    /// # Errors
1263    ///
1264    /// Returns an error if the collect event processing fails or if the fee collection creation fails.
1265    pub fn process_pool_collect_event(
1266        &self,
1267        collect_event: &CollectEvent,
1268        pool: &SharedPool,
1269        dex_extended: &DexExtended,
1270    ) -> anyhow::Result<PoolFeeCollect> {
1271        let timestamp = self
1272            .cache
1273            .get_block_timestamp(collect_event.block_number)
1274            .copied()
1275            .context("missing block timestamp for collect event")?;
1276
1277        let mut fee_collect = collect_event.to_pool_fee_collect(
1278            Arc::clone(&self.chain),
1279            Arc::clone(&dex_extended.dex),
1280            pool.instrument_id,
1281            timestamp,
1282        );
1283        fee_collect.block_hash = Some(self.observed_block_hash(
1284            collect_event.block_number,
1285            collect_event.block_hash,
1286            "collect",
1287        )?);
1288
1289        Ok(fee_collect)
1290    }
1291
1292    /// Processes a pool flash event and converts it to a flash loan.
1293    ///
1294    /// # Errors
1295    ///
1296    /// Returns an error if the flash event processing fails or if the flash loan creation fails.
1297    pub fn process_pool_flash_event(
1298        &self,
1299        flash_event: &FlashEvent,
1300        pool: &SharedPool,
1301    ) -> anyhow::Result<PoolFlash> {
1302        let timestamp = self
1303            .cache
1304            .get_block_timestamp(flash_event.block_number)
1305            .copied()
1306            .context("missing block timestamp for flash event")?;
1307
1308        let mut flash =
1309            flash_event.to_pool_flash(Arc::clone(&self.chain), pool.instrument_id, timestamp);
1310        flash.block_hash = Some(self.observed_block_hash(
1311            flash_event.block_number,
1312            flash_event.block_hash,
1313            "flash",
1314        )?);
1315
1316        Ok(flash)
1317    }
1318
1319    /// Processes a `SetFeeProtocol` event and converts it to a pool fee-protocol update.
1320    ///
1321    /// # Errors
1322    ///
1323    /// Returns an error if the event's block timestamp or cached block hash
1324    /// is missing from the cache, or if its observed block hash does not match
1325    /// the cached hash.
1326    pub fn process_pool_fee_protocol_update_event(
1327        &self,
1328        fee_protocol_update_event: &FeeProtocolUpdateEvent,
1329        pool: &SharedPool,
1330    ) -> anyhow::Result<PoolFeeProtocolUpdate> {
1331        let timestamp = self
1332            .cache
1333            .get_block_timestamp(fee_protocol_update_event.block_number)
1334            .copied()
1335            .context("missing block timestamp for SetFeeProtocol event")?;
1336
1337        let mut update = fee_protocol_update_event.to_pool_fee_protocol_update(
1338            Arc::clone(&self.chain),
1339            pool.instrument_id,
1340            timestamp,
1341        );
1342        update.block_hash = Some(self.observed_block_hash(
1343            fee_protocol_update_event.block_number,
1344            fee_protocol_update_event.block_hash,
1345            "SetFeeProtocol",
1346        )?);
1347
1348        Ok(update)
1349    }
1350
1351    /// Processes a `CollectProtocol` event and converts it to a pool protocol-fee withdrawal.
1352    ///
1353    /// # Errors
1354    ///
1355    /// Returns an error if the event's block timestamp or cached block hash
1356    /// is missing from the cache, or if its observed block hash does not match
1357    /// the cached hash.
1358    pub fn process_pool_fee_protocol_collect_event(
1359        &self,
1360        fee_protocol_collect_event: &FeeProtocolCollectEvent,
1361        pool: &SharedPool,
1362    ) -> anyhow::Result<PoolFeeProtocolCollect> {
1363        let timestamp = self
1364            .cache
1365            .get_block_timestamp(fee_protocol_collect_event.block_number)
1366            .copied()
1367            .context("missing block timestamp for CollectProtocol event")?;
1368
1369        let mut collect = fee_protocol_collect_event.to_pool_fee_protocol_collect(
1370            Arc::clone(&self.chain),
1371            pool.instrument_id,
1372            timestamp,
1373        );
1374        collect.block_hash = Some(self.observed_block_hash(
1375            fee_protocol_collect_event.block_number,
1376            fee_protocol_collect_event.block_hash,
1377            "CollectProtocol",
1378        )?);
1379
1380        Ok(collect)
1381    }
1382
1383    fn observed_block_hash(
1384        &self,
1385        block_number: u64,
1386        observed_hash: Option<B256>,
1387        event: &str,
1388    ) -> anyhow::Result<String> {
1389        let cached_hash = self.cache.get_block_hash(block_number).with_context(|| {
1390            format!("missing block hash for {event} event at block {block_number}")
1391        })?;
1392
1393        if !block_hash_matches(observed_hash, cached_hash) {
1394            anyhow::bail!(
1395                "block hash mismatch for {event} event at block {block_number}: observed {}, cached {cached_hash}",
1396                observed_hash.map_or_else(|| "none".to_string(), |hash| hash.to_string())
1397            );
1398        }
1399
1400        Ok(cached_hash.to_string())
1401    }
1402
1403    /// Synchronizes all pools and their tokens for a specific DEX within the given block range.
1404    ///
1405    /// This method performs a full sync of:
1406    /// 1. Pool creation events from the DEX factory
1407    /// 2. Token metadata for all tokens in discovered pools
1408    /// 3. Pool entities with proper token associations
1409    ///
1410    /// # Errors
1411    ///
1412    /// Returns an error if syncing pools, tokens, or DEX operations fail.
1413    pub async fn sync_exchange_pools(
1414        &mut self,
1415        dex: &DexType,
1416        from_block: u64,
1417        to_block: Option<u64>,
1418        reset: bool,
1419    ) -> anyhow::Result<()> {
1420        let dex_extended = self.get_dex_extended(dex)?.clone();
1421
1422        let mut service = PoolDiscoveryService::new(
1423            Arc::clone(&self.chain),
1424            &mut self.cache,
1425            &self.tokens,
1426            &self.hypersync_client,
1427            self.cancellation_token.clone(),
1428            self.config.clone(),
1429        );
1430
1431        service
1432            .sync_pools(&dex_extended, from_block, to_block, reset)
1433            .await?;
1434
1435        Ok(())
1436    }
1437
1438    /// Registers a decentralized exchange for data collection and event monitoring.
1439    ///
1440    /// Registration involves:
1441    /// 1. Adding the DEX to the cache
1442    /// 2. Loading existing pools for the DEX
1443    /// 3. Configuring event signatures for subscriptions
1444    ///
1445    /// # Errors
1446    ///
1447    /// Returns an error if DEX registration, cache operations, or pool loading fails.
1448    pub async fn register_dex_exchange(&mut self, dex_id: DexType) -> anyhow::Result<()> {
1449        self.register_dex(dex_id).await?;
1450        let _ = self.cache.load_pools(&dex_id).await?;
1451        Ok(())
1452    }
1453
1454    /// Registers a decentralized exchange but loads only a single pool into the cache.
1455    ///
1456    /// Like [`Self::register_dex_exchange`], but loads just `pool_identifier` instead of the whole
1457    /// DEX pool set, so per-pool tools (e.g. `analyze-pool`) avoid the full pool-set load. A pool
1458    /// absent from the cache database is left for the caller's later lookup to report.
1459    ///
1460    /// # Errors
1461    ///
1462    /// Returns an error if DEX registration or the pool load fails.
1463    pub async fn register_dex_exchange_for_pool(
1464        &mut self,
1465        dex_id: DexType,
1466        pool_identifier: &PoolIdentifier,
1467    ) -> anyhow::Result<()> {
1468        self.register_dex(dex_id).await?;
1469        let _ = self.cache.load_pool(&dex_id, pool_identifier).await?;
1470        Ok(())
1471    }
1472
1473    /// Registers a DEX in the cache and its event signatures for subscriptions, without loading pools.
1474    async fn register_dex(&mut self, dex_id: DexType) -> anyhow::Result<()> {
1475        let Some(dex_extended) = get_dex_extended(self.chain.name, &dex_id) else {
1476            anyhow::bail!("Unknown DEX {dex_id} on chain {}", self.chain.name);
1477        };
1478
1479        log::debug!("Registering DEX {dex_id} on chain {}", self.chain.name);
1480        self.cache.add_dex(Arc::clone(&dex_extended.dex)).await?;
1481        self.subscription_manager.register_dex_for_subscriptions(
1482            dex_id,
1483            dex_extended.swap_created_event.as_ref(),
1484            dex_extended.mint_created_event.as_ref(),
1485            dex_extended.burn_created_event.as_ref(),
1486            dex_extended.collect_created_event.as_ref(),
1487            dex_extended.flash_created_event.as_deref(),
1488        );
1489        self.subscription_manager.register_dex_fee_protocol_events(
1490            dex_id,
1491            dex_extended.fee_protocol_update_event.as_deref(),
1492            dex_extended.fee_protocol_collect_event.as_deref(),
1493        );
1494        Ok(())
1495    }
1496
1497    /// Bootstraps a [`PoolProfiler`] with the latest state for a given pool.
1498    ///
1499    /// Uses two paths depending on whether the pool has been synced to the database:
1500    /// - **Never synced**: Streams events from HyperSync, restores from on-chain RPC, returns `(profiler, true)`.
1501    /// - **Previously synced**: Syncs new events to DB, streams from DB, returns `(profiler, false)`.
1502    ///
1503    /// Both paths restore from the latest valid snapshot first (if available), otherwise initialize with pool's initial price.
1504    ///
1505    /// # Returns
1506    ///
1507    /// - `PoolProfiler`: Hydrated profiler with current pool state
1508    /// - `bool`: `true` if constructed from RPC (already valid), `false` if from DB (needs validation)
1509    ///
1510    /// # Errors
1511    ///
1512    /// Returns an error if database is not initialized or event processing fails.
1513    ///
1514    /// # Panics
1515    ///
1516    /// Panics if the database reference is unavailable.
1517    pub async fn bootstrap_latest_pool_profiler(
1518        &mut self,
1519        pool: &SharedPool,
1520        to_block: Option<u64>,
1521    ) -> anyhow::Result<(PoolProfiler, bool)> {
1522        log::debug!(
1523            "Bootstrapping latest pool profiler for pool {}",
1524            pool.address
1525        );
1526
1527        if self.cache.database.is_none() {
1528            anyhow::bail!(
1529                "Database is not initialized, so we cannot properly bootstrap the latest pool profiler"
1530            );
1531        }
1532
1533        let to_block = match to_block {
1534            Some(block) => block,
1535            None => self.hypersync_client.current_block().await,
1536        };
1537        let (mut profiler, from_position) = self
1538            .seed_pool_profiler_from_latest_snapshot(pool, to_block)
1539            .await?;
1540
1541        // If we don't have never synced pool events, proceed with faster
1542        // construction of pool profiler from hypersync and RPC, where we
1543        // dont need syncing of pool events and fetching it from database
1544        if self
1545            .cache
1546            .database
1547            .as_ref()
1548            .unwrap()
1549            .get_pool_last_synced_block(self.chain.chain_id, &pool.dex.name, &pool.pool_identifier)
1550            .await?
1551            .is_none()
1552        {
1553            return self
1554                .construct_pool_profiler_from_hypersync_rpc(profiler, from_position, to_block)
1555                .await;
1556        }
1557
1558        // Sync the pool events before bootstrapping of pool profiler
1559        self.sync_pool_events(
1560            &pool.dex.name,
1561            pool.pool_identifier,
1562            None,
1563            Some(to_block),
1564            false,
1565        )
1566        .await
1567        .context("failed to sync pool events for snapshot request")?;
1568
1569        if !profiler.is_initialized {
1570            if let Some(initial_sqrt_price_x96) = pool.initial_sqrt_price_x96 {
1571                profiler.initialize(initial_sqrt_price_x96)?;
1572            } else {
1573                anyhow::bail!(
1574                    "Pool is not initialized and it doesn't contain initial price, cannot bootstrap profiler"
1575                );
1576            }
1577        }
1578
1579        let from_block = from_position
1580            .as_ref()
1581            .map_or(profiler.pool.creation_block, |block_position| {
1582                block_position.number
1583            });
1584        let total_blocks = to_block.saturating_sub(from_block) + 1;
1585
1586        // Enable embedded profiler reporting
1587        profiler.enable_reporting(from_block, total_blocks, BLOCKS_PROCESS_IN_SYNC_REPORT);
1588
1589        let mut stream = self.cache.database.as_ref().unwrap().stream_pool_events(
1590            Arc::clone(&pool.chain),
1591            Arc::clone(&pool.dex),
1592            pool.instrument_id,
1593            pool.pool_identifier,
1594            from_position.clone(),
1595            Some(to_block),
1596        );
1597
1598        while let Some(result) = stream.next().await {
1599            match result {
1600                Ok(event) => {
1601                    profiler.process(&event)?;
1602                }
1603                Err(e) => return Err(e).context("failed to stream pool event from database"),
1604            }
1605        }
1606
1607        profiler.finalize_reporting();
1608
1609        Ok((profiler, false))
1610    }
1611
1612    /// Bootstraps a pool profiler by reading liquidity topology from HyperSync and state from RPC.
1613    ///
1614    /// This mode avoids storing the full swap history. It streams Initialize, Mint, Burn, and
1615    /// fee-protocol updates only, then hydrates the exact target block from the pool contract.
1616    ///
1617    /// # Errors
1618    ///
1619    /// Returns an error if database is not initialized, event streaming fails, or RPC hydration fails.
1620    pub async fn bootstrap_pool_profiler_from_rpc_snapshot(
1621        &mut self,
1622        pool: &SharedPool,
1623        to_block: u64,
1624    ) -> anyhow::Result<(PoolProfiler, bool)> {
1625        if self.cache.database.is_none() {
1626            anyhow::bail!(
1627                "Database is not initialized, so we cannot bootstrap the pool profiler from an RPC snapshot"
1628            );
1629        }
1630
1631        self.construct_pool_profiler_from_hypersync_rpc(
1632            PoolProfiler::new(Arc::clone(pool)),
1633            None,
1634            to_block,
1635        )
1636        .await
1637    }
1638
1639    /// Advances an RPC-hydrated profiler to a later checkpoint.
1640    ///
1641    /// The profiler must come from [`Self::bootstrap_pool_profiler_from_rpc_snapshot`] or an earlier
1642    /// call to this method. This keeps one command incremental without trusting an unproven stored
1643    /// snapshot as the topology source.
1644    ///
1645    /// # Errors
1646    ///
1647    /// Returns an error if the profiler has no RPC snapshot watermark, the target precedes that
1648    /// watermark, event streaming fails, or RPC hydration fails.
1649    pub async fn advance_pool_profiler_from_rpc_snapshot(
1650        &mut self,
1651        profiler: PoolProfiler,
1652        to_block: u64,
1653    ) -> anyhow::Result<(PoolProfiler, bool)> {
1654        let from_position = profiler.last_processed_event.clone().ok_or_else(|| {
1655            anyhow::anyhow!("cannot advance an RPC profiler without a snapshot watermark")
1656        })?;
1657
1658        if to_block < from_position.number {
1659            anyhow::bail!(
1660                "cannot advance RPC profiler from block {} to earlier block {to_block}",
1661                from_position.number
1662            );
1663        }
1664
1665        self.construct_pool_profiler_from_hypersync_rpc(profiler, Some(from_position), to_block)
1666            .await
1667    }
1668
1669    async fn seed_pool_profiler_from_latest_snapshot(
1670        &self,
1671        pool: &SharedPool,
1672        to_block: u64,
1673    ) -> anyhow::Result<(PoolProfiler, Option<BlockPosition>)> {
1674        let mut profiler = PoolProfiler::new(Arc::clone(pool));
1675
1676        let from_position = match self
1677            .cache
1678            .database
1679            .as_ref()
1680            .expect("database presence is checked by caller")
1681            .load_latest_pool_snapshot(
1682                pool.chain.chain_id,
1683                &pool.pool_identifier,
1684                Some(to_block),
1685                true,
1686            )
1687            .await
1688        {
1689            Ok(Some(snapshot)) => {
1690                // Empty snapshots at the pool's creation block are stubs left behind by an
1691                // earlier bootstrap that bailed before any liquidity events landed. Restoring
1692                // marks the profiler as initialized, which then conflicts with the Initialize
1693                // event that hypersync re-emits at the same block. Fall through to a fresh
1694                // bootstrap rather than trust the stub.
1695                if snapshot.positions.is_empty()
1696                    && snapshot.ticks.is_empty()
1697                    && snapshot.block_position.number == pool.creation_block
1698                {
1699                    log::warn!(
1700                        "Ignoring empty stub snapshot at pool creation block {} for {}; rebuilding from events",
1701                        snapshot.block_position.number.separate_with_commas(),
1702                        pool.instrument_id,
1703                    );
1704                    None
1705                } else {
1706                    log::debug!(
1707                        "Loaded valid snapshot from block {} which contains {} positions and {} ticks",
1708                        snapshot.block_position.number.separate_with_commas(),
1709                        snapshot.positions.len(),
1710                        snapshot.ticks.len()
1711                    );
1712                    let block_position = snapshot.block_position.clone();
1713                    profiler.restore_from_snapshot(snapshot)?;
1714                    log::debug!("Restored profiler from snapshot");
1715                    Some(block_position)
1716                }
1717            }
1718            _ => {
1719                log::debug!("No valid snapshot found, processing from beginning");
1720                None
1721            }
1722        };
1723
1724        Ok((profiler, from_position))
1725    }
1726
1727    /// Constructs a pool profiler by fetching events directly from HyperSync RPC.
1728    ///
1729    /// This method is used when the pool has never been synced to the database. It streams
1730    /// liquidity events (mints, burns) directly from HyperSync and processes them
1731    /// to build up the profiler's state in real-time. After processing all events, it
1732    /// restores the profiler from on-chain state at the target block with the provided ticks and
1733    /// positions.
1734    ///
1735    /// # Returns
1736    ///
1737    /// Returns a tuple of:
1738    /// - `PoolProfiler`: The hydrated profiler with state built from events
1739    /// - `bool`: Always `true` to indicate the profiler state was valid, and it was constructed from RPC
1740    ///
1741    /// # Errors
1742    ///
1743    /// Returns an error if:
1744    /// - Event streaming from HyperSync fails.
1745    /// - Event parsing or processing fails.
1746    /// - DEX configuration is invalid.
1747    async fn construct_pool_profiler_from_hypersync_rpc(
1748        &mut self,
1749        mut profiler: PoolProfiler,
1750        from_position: Option<BlockPosition>,
1751        to_block: u64,
1752    ) -> anyhow::Result<(PoolProfiler, bool)> {
1753        log::debug!("Constructing pool profiler from hypersync stream and RPC target block state");
1754        let dex_extended = self.get_dex_extended(&profiler.pool.dex.name)?.clone();
1755        let mint_event_signature = dex_extended.mint_created_event.as_ref();
1756        let burn_event_signature = dex_extended.burn_created_event.as_ref();
1757        let initialize_event_signature =
1758            if let Some(initialize_event) = &dex_extended.initialize_event {
1759                initialize_event.as_ref()
1760            } else {
1761                anyhow::bail!(
1762                    "DEX {} does not have initialize event set.",
1763                    profiler.pool.dex.name
1764                );
1765            };
1766        let mint_sig_bytes = hex::decode(
1767            mint_event_signature
1768                .strip_prefix("0x")
1769                .unwrap_or(mint_event_signature),
1770        )?;
1771        let burn_sig_bytes = hex::decode(
1772            burn_event_signature
1773                .strip_prefix("0x")
1774                .unwrap_or(burn_event_signature),
1775        )?;
1776        let initialize_sig_bytes = hex::decode(
1777            initialize_event_signature
1778                .strip_prefix("0x")
1779                .unwrap_or(initialize_event_signature),
1780        )?;
1781        let protocol_update_event_signature = dex_extended.fee_protocol_update_event.as_deref();
1782        let protocol_update_sig_bytes = protocol_update_event_signature
1783            .map(|s| hex::decode(s.strip_prefix("0x").unwrap_or(s)).unwrap_or_default());
1784        let protocol_collect_event_signature = dex_extended.fee_protocol_collect_event.as_deref();
1785        let protocol_collect_sig_bytes = protocol_collect_event_signature
1786            .map(|s| hex::decode(s.strip_prefix("0x").unwrap_or(s)).unwrap_or_default());
1787
1788        let from_block = from_position.map_or(profiler.pool.creation_block, |block_position| {
1789            block_position.number
1790        });
1791        let total_blocks = to_block.saturating_sub(from_block) + 1;
1792
1793        log::debug!(
1794            "Bootstrapping pool profiler for pool {} from block {} to {} (total: {} blocks)",
1795            profiler.pool.address,
1796            from_block.separate_with_commas(),
1797            to_block.separate_with_commas(),
1798            total_blocks.separate_with_commas()
1799        );
1800
1801        // Enable embedded profiler reporting
1802        profiler.enable_reporting(from_block, total_blocks, BLOCKS_PROCESS_IN_SYNC_REPORT);
1803
1804        let mut event_signatures = vec![
1805            mint_event_signature,
1806            burn_event_signature,
1807            initialize_event_signature,
1808        ];
1809
1810        if let Some(event) = protocol_update_event_signature {
1811            event_signatures.push(event);
1812        }
1813
1814        if let Some(event) = protocol_collect_event_signature {
1815            event_signatures.push(event);
1816        }
1817
1818        let pool_events_stream = self
1819            .hypersync_client
1820            .request_contract_events_stream(
1821                from_block,
1822                Some(to_block),
1823                &profiler.pool.address,
1824                event_signatures,
1825            )
1826            .await;
1827        tokio::pin!(pool_events_stream);
1828        let mut block_batch: Vec<Block> = Vec::with_capacity(POOL_EVENT_BLOCK_BATCH_SIZE);
1829
1830        while let Some(item) = pool_events_stream.next().await {
1831            let log = match item {
1832                PoolEventStreamItem::Block(block) => {
1833                    self.record_pool_event_block(block, &mut block_batch)
1834                        .await?;
1835                    continue;
1836                }
1837                PoolEventStreamItem::Log(log) => log,
1838            };
1839            let event_sig_bytes = extract_event_signature_bytes(&log)?;
1840
1841            if event_sig_bytes == initialize_sig_bytes {
1842                if profiler.is_initialized {
1843                    // Profiler was restored from a snapshot at or after this block; the
1844                    // initialize state is already in place. Skip the re-init that would
1845                    // otherwise trip AlreadyInitialized.
1846                    log::debug!(
1847                        "Profiler already initialized; skipping Initialize event at block {}",
1848                        extract_block_number(&log)?.separate_with_commas(),
1849                    );
1850                } else {
1851                    let initialize_event = dex_extended.parse_initialize_event_hypersync(&log)?;
1852                    profiler.initialize(initialize_event.sqrt_price_x96)?;
1853                    self.cache
1854                        .database
1855                        .as_ref()
1856                        .unwrap()
1857                        .update_pool_initial_price_tick(self.chain.chain_id, &initialize_event)
1858                        .await?;
1859                }
1860            } else if event_sig_bytes == mint_sig_bytes {
1861                let mint_event = dex_extended.parse_mint_event_hypersync(&log)?;
1862                let liquidity_update = self
1863                    .process_pool_mint_event(&mint_event, &profiler.pool, &dex_extended)
1864                    .with_context(|| {
1865                        format!(
1866                            "failed to process mint event at block {}",
1867                            mint_event.block_number
1868                        )
1869                    })?;
1870                profiler.process(&DexPoolData::LiquidityUpdate(liquidity_update))?;
1871            } else if event_sig_bytes == burn_sig_bytes {
1872                let burn_event = dex_extended.parse_burn_event_hypersync(&log)?;
1873                let liquidity_update = self
1874                    .process_pool_burn_event(&burn_event, &profiler.pool, &dex_extended)
1875                    .with_context(|| {
1876                        format!(
1877                            "failed to process burn event at block {}",
1878                            burn_event.block_number
1879                        )
1880                    })?;
1881                profiler.process(&DexPoolData::LiquidityUpdate(liquidity_update))?;
1882            } else if protocol_update_sig_bytes
1883                .as_ref()
1884                .is_some_and(|sig| sig.as_slice() == event_sig_bytes)
1885            {
1886                let fee_protocol_update_event =
1887                    dex_extended.parse_fee_protocol_update_event_hypersync(&log)?;
1888                let update = self
1889                    .process_pool_fee_protocol_update_event(
1890                        &fee_protocol_update_event,
1891                        &profiler.pool,
1892                    )
1893                    .with_context(|| {
1894                        format!(
1895                            "failed to process SetFeeProtocol event at block {}",
1896                            fee_protocol_update_event.block_number
1897                        )
1898                    })?;
1899                profiler.process(&DexPoolData::FeeProtocolUpdate(update))?;
1900            } else if protocol_collect_sig_bytes
1901                .as_ref()
1902                .is_some_and(|sig| sig.as_slice() == event_sig_bytes)
1903            {
1904                let fee_protocol_collect_event =
1905                    dex_extended.parse_fee_protocol_collect_event_hypersync(&log)?;
1906                let collect = self
1907                    .process_pool_fee_protocol_collect_event(
1908                        &fee_protocol_collect_event,
1909                        &profiler.pool,
1910                    )
1911                    .with_context(|| {
1912                        format!(
1913                            "failed to process CollectProtocol event at block {}",
1914                            fee_protocol_collect_event.block_number
1915                        )
1916                    })?;
1917                profiler.process(&DexPoolData::FeeProtocolCollect(collect))?;
1918            } else {
1919                let event_signature = hex::encode(event_sig_bytes);
1920                anyhow::bail!(
1921                    "unexpected event signature in bootstrap_latest_pool_profiler: {event_signature} for log {log:?}"
1922                );
1923            }
1924        }
1925
1926        self.flush_pool_event_blocks(&mut block_batch).await?;
1927        profiler.finalize_reporting();
1928
1929        let snapshot_block_position = self.block_scoped_snapshot_position(to_block).await?;
1930        let on_chain_snapshot = self
1931            .get_on_chain_snapshot_at_position(&profiler, snapshot_block_position)
1932            .await
1933            .with_context(|| {
1934                format!(
1935                    "failed to restore pool {} from RPC snapshot at target block {} with {} ticks and {} positions",
1936                    profiler.pool.address,
1937                    to_block.separate_with_commas(),
1938                    profiler.get_active_tick_values().len().separate_with_commas(),
1939                    profiler.get_all_position_keys().len().separate_with_commas()
1940                )
1941            })?;
1942        Self::validate_rpc_snapshot_topology(&profiler, &on_chain_snapshot).with_context(|| {
1943            format!(
1944                "RPC snapshot topology validation failed for pool {} at block {}",
1945                profiler.pool.address,
1946                to_block.separate_with_commas()
1947            )
1948        })?;
1949        profiler.restore_from_snapshot(on_chain_snapshot)?;
1950
1951        Ok((profiler, true))
1952    }
1953
1954    /// Validates a pool profiler's state against on-chain data for accuracy verification.
1955    ///
1956    /// This method performs integrity checking by comparing the profiler's internal state
1957    /// (positions, ticks, liquidity) with the actual on-chain smart contract state. For UniswapV3
1958    /// pools, it fetches current on-chain data and verifies that the profiler's tracked state matches.
1959    /// Returns [`SnapshotValidation::OnChain`] when the profiler matches on-chain state,
1960    /// [`SnapshotValidation::Invalid`] when it does not, and [`SnapshotValidation::Replay`] when the
1961    /// on-chain state could not be fetched (e.g. a non-archive RPC for a historical block); in the
1962    /// last case the replay-derived snapshot is kept. The resolved state is persisted for the
1963    /// `OnChain` and `Invalid` outcomes; `Replay` leaves the snapshot at its inserted default so a
1964    /// transient RPC failure cannot clobber a prior definitive verdict.
1965    ///
1966    /// # Errors
1967    ///
1968    /// Returns an error if database operations fail when persisting the validation state.
1969    ///
1970    /// # Panics
1971    ///
1972    /// Panics if the profiler does not have a last_processed_event when already_validated is true.
1973    pub async fn check_snapshot_validity(
1974        &self,
1975        profiler: &PoolProfiler,
1976        already_validated: bool,
1977    ) -> anyhow::Result<SnapshotValidation> {
1978        let (validation, block_position) = if already_validated {
1979            // Skip RPC call - profiler was validated during construction from RPC
1980            log::debug!("Snapshot already validated from RPC, skipping on-chain comparison");
1981            let last_event = profiler
1982                .last_processed_event
1983                .clone()
1984                .expect("Profiler should have last_processed_event");
1985            (SnapshotValidation::OnChain, Some(last_event))
1986        } else {
1987            // Fetch on-chain state and compare
1988            match self.get_on_chain_snapshot(profiler).await {
1989                Ok(on_chain_snapshot) => {
1990                    log::debug!("Comparing profiler state with on-chain state...");
1991                    let comparison = compare_pool_profiler_detailed(profiler, &on_chain_snapshot);
1992                    let validation = if comparison.is_valid_for_snapshot() {
1993                        if !comparison.is_exact_match() {
1994                            log::warn!(
1995                                "Pool profiler snapshot has a non-structural mismatch (sqrt ratio, fee protocol, or protocol fees); accepting snapshot"
1996                            );
1997                        }
1998                        SnapshotValidation::OnChain
1999                    } else {
2000                        log::error!(
2001                            "Pool profiler state does NOT match on-chain smart contract state"
2002                        );
2003                        SnapshotValidation::Invalid
2004                    };
2005                    (validation, Some(on_chain_snapshot.block_position))
2006                }
2007                Err(e) => {
2008                    log::warn!(
2009                        "Could not validate snapshot against on-chain state, keeping replay-derived snapshot: {e}"
2010                    );
2011                    // RPC could not reach the block. Report any stored verdict so stdout agrees with
2012                    // a pre-existing on_chain/invalid row; the None block position below skips the
2013                    // persist step, so a transient failure cannot clobber that verdict.
2014                    let reported = self
2015                        .stored_snapshot_validation(profiler)
2016                        .await?
2017                        .unwrap_or(SnapshotValidation::Replay);
2018                    (reported, None)
2019                }
2020            }
2021        };
2022
2023        if let (Some(block_position), Some(cache_database)) = (block_position, &self.cache.database)
2024        {
2025            cache_database
2026                .set_pool_snapshot_validation_state(
2027                    profiler.pool.chain.chain_id,
2028                    &profiler.pool.pool_identifier,
2029                    block_position.number,
2030                    block_position.transaction_index,
2031                    block_position.log_index,
2032                    validation.as_str(),
2033                )
2034                .await?;
2035            log::debug!(
2036                "Set pool snapshot validation state to {}",
2037                validation.as_str()
2038            );
2039        }
2040
2041        Ok(validation)
2042    }
2043
2044    /// Reads the persisted [`SnapshotValidation`] for the profiler's current snapshot watermark.
2045    ///
2046    /// Returns `None` when no database is configured, the profiler has no processed event, or no
2047    /// snapshot row exists at that watermark.
2048    async fn stored_snapshot_validation(
2049        &self,
2050        profiler: &PoolProfiler,
2051    ) -> anyhow::Result<Option<SnapshotValidation>> {
2052        let (Some(block_position), Some(cache_database)) =
2053            (profiler.last_processed_event.as_ref(), &self.cache.database)
2054        else {
2055            return Ok(None);
2056        };
2057
2058        let stored = cache_database
2059            .get_pool_snapshot_validation_state(
2060                profiler.pool.chain.chain_id,
2061                &profiler.pool.pool_identifier,
2062                block_position.number,
2063                block_position.transaction_index,
2064                block_position.log_index,
2065            )
2066            .await?;
2067
2068        Ok(stored.and_then(|token| SnapshotValidation::from_db_token(&token)))
2069    }
2070
2071    /// Fetches current on-chain pool state at the last processed block.
2072    ///
2073    /// Queries the pool smart contract to retrieve active tick liquidity and position data,
2074    /// using the profiler's active positions and last processed block number.
2075    /// Used for profiler state restoration after bootstrapping and validation.
2076    async fn get_on_chain_snapshot(&self, profiler: &PoolProfiler) -> anyhow::Result<PoolSnapshot> {
2077        let last_processed_event = Self::last_processed_event_for_on_chain_snapshot(profiler)?;
2078        self.get_on_chain_snapshot_at_position(profiler, last_processed_event)
2079            .await
2080    }
2081
2082    async fn get_on_chain_snapshot_at_position(
2083        &self,
2084        profiler: &PoolProfiler,
2085        block_position: BlockPosition,
2086    ) -> anyhow::Result<PoolSnapshot> {
2087        // PancakeSwap V3 shares the Uniswap V3 pool read ABI except slot0.feeProtocol width.
2088        if matches!(
2089            profiler.pool.dex.name,
2090            DexType::UniswapV3 | DexType::PancakeSwapV3
2091        ) {
2092            let fee_protocol_encoding = match profiler.pool.dex.name {
2093                DexType::PancakeSwapV3 => FeeProtocolEncoding::PancakeSwapV3BasisPoints,
2094                _ => FeeProtocolEncoding::UniswapV3Packed,
2095            };
2096            let timestamp = Self::timestamp_for_on_chain_snapshot(
2097                profiler,
2098                self.cache
2099                    .get_block_timestamp(block_position.number)
2100                    .copied(),
2101            )?;
2102            let on_chain_snapshot = self
2103                .univ3_pool
2104                .fetch_snapshot(
2105                    &profiler.pool.address,
2106                    profiler.pool.instrument_id,
2107                    profiler.get_active_tick_values().as_slice(),
2108                    &profiler.get_all_position_keys(),
2109                    block_position,
2110                    timestamp, // ts_event
2111                    timestamp, // ts_init (same block timestamp)
2112                    fee_protocol_encoding,
2113                )
2114                .await?;
2115
2116            Ok(on_chain_snapshot)
2117        } else {
2118            anyhow::bail!(
2119                "Fetching on-chain snapshot for Dex protocol {} is not supported yet.",
2120                profiler.pool.dex.name
2121            )
2122        }
2123    }
2124
2125    fn validate_rpc_snapshot_topology(
2126        profiler: &PoolProfiler,
2127        snapshot: &PoolSnapshot,
2128    ) -> anyhow::Result<()> {
2129        let tick_spacing = i32::try_from(
2130            profiler
2131                .pool
2132                .tick_spacing
2133                .context("pool tick spacing is not set")?,
2134        )?;
2135        let expected_positions: AHashMap<_, _> = profiler
2136            .get_all_positions()
2137            .into_iter()
2138            .map(|position| {
2139                (
2140                    (position.owner, position.tick_lower, position.tick_upper),
2141                    position.liquidity,
2142                )
2143            })
2144            .collect();
2145        let actual_positions: AHashMap<_, _> = snapshot
2146            .positions
2147            .iter()
2148            .map(|position| {
2149                (
2150                    (position.owner, position.tick_lower, position.tick_upper),
2151                    position.liquidity,
2152                )
2153            })
2154            .collect();
2155
2156        if actual_positions.len() != snapshot.positions.len()
2157            || actual_positions != expected_positions
2158        {
2159            anyhow::bail!(
2160                "RPC positions do not match the complete HyperSync topology: expected {} positions, received {}",
2161                expected_positions.len(),
2162                actual_positions.len()
2163            );
2164        }
2165
2166        let actual_ticks: AHashMap<_, _> = snapshot
2167            .ticks
2168            .iter()
2169            .map(|tick| (tick.value, tick))
2170            .collect();
2171
2172        if actual_ticks.len() != snapshot.ticks.len() {
2173            anyhow::bail!("RPC snapshot contains duplicate ticks");
2174        }
2175
2176        let expected_tick_values = profiler.get_active_tick_values();
2177        if actual_ticks.len() != expected_tick_values.len() {
2178            anyhow::bail!(
2179                "RPC ticks do not match the complete HyperSync topology: expected {} ticks, received {}",
2180                expected_tick_values.len(),
2181                actual_ticks.len()
2182            );
2183        }
2184
2185        for tick_value in expected_tick_values {
2186            let expected_tick = profiler
2187                .get_tick(tick_value)
2188                .with_context(|| format!("missing replay tick {tick_value}"))?;
2189            let actual_tick = actual_ticks
2190                .get(&tick_value)
2191                .with_context(|| format!("RPC snapshot omitted tick {tick_value}"))?;
2192
2193            if !actual_tick.initialized
2194                || actual_tick.liquidity_gross == 0
2195                || actual_tick.liquidity_gross != expected_tick.liquidity_gross
2196                || actual_tick.liquidity_net != expected_tick.liquidity_net
2197            {
2198                anyhow::bail!(
2199                    "RPC tick {tick_value} topology mismatch: expected gross={} net={}, received gross={} net={} initialized={}",
2200                    expected_tick.liquidity_gross,
2201                    expected_tick.liquidity_net,
2202                    actual_tick.liquidity_gross,
2203                    actual_tick.liquidity_net,
2204                    actual_tick.initialized
2205                );
2206            }
2207        }
2208
2209        let mut derived_ticks: AHashMap<i32, (u128, i128)> = AHashMap::new();
2210        let mut active_liquidity = 0_u128;
2211
2212        for position in &snapshot.positions {
2213            if position.tick_lower >= position.tick_upper
2214                || position.tick_lower < PoolTick::MIN_TICK
2215                || position.tick_upper > PoolTick::MAX_TICK
2216                || position.tick_lower % tick_spacing != 0
2217                || position.tick_upper % tick_spacing != 0
2218            {
2219                anyhow::bail!(
2220                    "RPC position {} has invalid tick range [{}, {}) for spacing {tick_spacing}",
2221                    position.owner,
2222                    position.tick_lower,
2223                    position.tick_upper
2224                );
2225            }
2226
2227            if position.liquidity == 0 {
2228                continue;
2229            }
2230
2231            let liquidity_net = i128::try_from(position.liquidity).with_context(|| {
2232                format!(
2233                    "RPC position {} liquidity exceeds i128::MAX",
2234                    position.owner
2235                )
2236            })?;
2237            let lower = derived_ticks.entry(position.tick_lower).or_default();
2238            lower.0 = lower
2239                .0
2240                .checked_add(position.liquidity)
2241                .context("lower tick liquidity gross overflow")?;
2242            lower.1 = lower
2243                .1
2244                .checked_add(liquidity_net)
2245                .context("lower tick liquidity net overflow")?;
2246            let upper = derived_ticks.entry(position.tick_upper).or_default();
2247            upper.0 = upper
2248                .0
2249                .checked_add(position.liquidity)
2250                .context("upper tick liquidity gross overflow")?;
2251            upper.1 = upper
2252                .1
2253                .checked_sub(liquidity_net)
2254                .context("upper tick liquidity net underflow")?;
2255
2256            if position.tick_lower <= snapshot.state.current_tick
2257                && snapshot.state.current_tick < position.tick_upper
2258            {
2259                active_liquidity = active_liquidity
2260                    .checked_add(position.liquidity)
2261                    .context("active position liquidity overflow")?;
2262            }
2263        }
2264
2265        if active_liquidity != snapshot.state.liquidity {
2266            anyhow::bail!(
2267                "RPC active liquidity mismatch: positions sum to {active_liquidity}, global state reports {}",
2268                snapshot.state.liquidity
2269            );
2270        }
2271
2272        if derived_ticks.len() != actual_ticks.len() {
2273            anyhow::bail!(
2274                "RPC tick topology does not match positions: derived {} ticks, received {}",
2275                derived_ticks.len(),
2276                actual_ticks.len()
2277            );
2278        }
2279
2280        for (tick_value, (liquidity_gross, liquidity_net)) in derived_ticks {
2281            let actual_tick = actual_ticks
2282                .get(&tick_value)
2283                .with_context(|| format!("RPC snapshot omitted position boundary {tick_value}"))?;
2284
2285            if actual_tick.liquidity_gross != liquidity_gross
2286                || actual_tick.liquidity_net != liquidity_net
2287            {
2288                anyhow::bail!(
2289                    "RPC tick {tick_value} does not match positions: derived gross={liquidity_gross} net={liquidity_net}, received gross={} net={}",
2290                    actual_tick.liquidity_gross,
2291                    actual_tick.liquidity_net
2292                );
2293            }
2294        }
2295
2296        Ok(())
2297    }
2298
2299    fn timestamp_for_on_chain_snapshot(
2300        profiler: &PoolProfiler,
2301        cached_timestamp: Option<UnixNanos>,
2302    ) -> anyhow::Result<UnixNanos> {
2303        cached_timestamp
2304            .or(profiler.last_processed_ts)
2305            .context("missing block timestamp for on-chain snapshot")
2306    }
2307
2308    fn last_processed_event_for_on_chain_snapshot(
2309        profiler: &PoolProfiler,
2310    ) -> anyhow::Result<BlockPosition> {
2311        profiler.last_processed_event.clone().with_context(|| {
2312            format!(
2313                "cannot fetch on-chain snapshot for pool {} without a processed event",
2314                profiler.pool.address
2315            )
2316        })
2317    }
2318
2319    async fn block_scoped_snapshot_position(
2320        &mut self,
2321        block_number: u64,
2322    ) -> anyhow::Result<BlockPosition> {
2323        let blocks_stream = self
2324            .hypersync_client
2325            .request_blocks_stream(block_number, Some(block_number))
2326            .await;
2327        tokio::pin!(blocks_stream);
2328        let block = blocks_stream
2329            .next()
2330            .await
2331            .with_context(|| format!("failed to fetch block {block_number} for RPC snapshot"))?;
2332
2333        let block_position =
2334            Self::block_scoped_snapshot_position_from_block(&mut self.cache, &block, block_number)?;
2335        self.cache.add_pool_event_blocks_batch(vec![block]).await?;
2336
2337        Ok(block_position)
2338    }
2339
2340    fn block_scoped_snapshot_position_from_block(
2341        cache: &mut BlockchainCache,
2342        block: &Block,
2343        block_number: u64,
2344    ) -> anyhow::Result<BlockPosition> {
2345        if block.number != block_number {
2346            anyhow::bail!(
2347                "Fetched block {} while requesting RPC snapshot block {}",
2348                block.number,
2349                block_number
2350            );
2351        }
2352
2353        cache.cache_block_metadata(block)?;
2354
2355        Ok(BlockPosition::new(
2356            block.number,
2357            block.hash.clone(),
2358            BLOCK_SCOPED_SNAPSHOT_INDEX,
2359            BLOCK_SCOPED_SNAPSHOT_INDEX,
2360        )
2361        .with_block_hash(Some(block.hash.clone())))
2362    }
2363
2364    /// Replays historical events for a pool to hydrate its profiler state.
2365    ///
2366    /// Streams all historical swap, liquidity, and fee collect events from the database
2367    /// and sends them through the normal data event pipeline to build up pool profiler state.
2368    ///
2369    /// # Errors
2370    ///
2371    /// Returns an error if database streaming fails or event processing fails.
2372    pub async fn replay_pool_events(&self, pool: &Pool, dex: &SharedDex) -> anyhow::Result<()> {
2373        if let Some(database) = &self.cache.database {
2374            log::debug!(
2375                "Replaying historical events for pool {} to hydrate profiler",
2376                pool.instrument_id
2377            );
2378
2379            let mut event_stream = database.stream_pool_events(
2380                Arc::clone(&self.chain),
2381                Arc::clone(dex),
2382                pool.instrument_id,
2383                pool.pool_identifier,
2384                None,
2385                None,
2386            );
2387            let mut event_count = 0;
2388
2389            while let Some(event_result) = event_stream.next().await {
2390                match event_result {
2391                    Ok(event) => {
2392                        let data_event = match event {
2393                            DexPoolData::Swap(swap) => DataEvent::DeFi(DefiData::PoolSwap(swap)),
2394                            DexPoolData::LiquidityUpdate(update) => {
2395                                DataEvent::DeFi(DefiData::PoolLiquidityUpdate(update))
2396                            }
2397                            DexPoolData::FeeCollect(collect) => {
2398                                DataEvent::DeFi(DefiData::PoolFeeCollect(collect))
2399                            }
2400                            DexPoolData::FeeProtocolUpdate(update) => {
2401                                DataEvent::DeFi(DefiData::PoolFeeProtocolUpdate(update))
2402                            }
2403                            DexPoolData::FeeProtocolCollect(collect) => {
2404                                DataEvent::DeFi(DefiData::PoolFeeProtocolCollect(collect))
2405                            }
2406                            DexPoolData::Flash(flash) => {
2407                                DataEvent::DeFi(DefiData::PoolFlash(flash))
2408                            }
2409                        };
2410                        self.send_data(data_event);
2411                        event_count += 1;
2412                    }
2413                    Err(e) => {
2414                        log::error!("Error streaming event for pool {}: {e}", pool.instrument_id);
2415                    }
2416                }
2417            }
2418
2419            log::debug!(
2420                "Replayed {event_count} historical events for pool {}",
2421                pool.instrument_id
2422            );
2423        } else {
2424            log::debug!(
2425                "No database available, skipping event replay for pool {}",
2426                pool.instrument_id
2427            );
2428        }
2429
2430        Ok(())
2431    }
2432
2433    /// Determines the starting block for syncing operations.
2434    fn determine_from_block(&self) -> u64 {
2435        self.config
2436            .from_block
2437            .unwrap_or_else(|| self.cache.min_dex_creation_block().unwrap_or(0))
2438    }
2439
2440    /// Retrieves extended DEX information for a registered DEX.
2441    fn get_dex_extended(&self, dex_id: &DexType) -> anyhow::Result<&DexExtended> {
2442        if !self.cache.get_registered_dexes().contains(dex_id) {
2443            anyhow::bail!("DEX {dex_id} is not registered in the data client");
2444        }
2445
2446        match get_dex_extended(self.chain.name, dex_id) {
2447            Some(dex) => Ok(dex),
2448            None => anyhow::bail!("Dex {dex_id} doesn't exist for chain {}", self.chain.name),
2449        }
2450    }
2451
2452    /// Retrieves a pool from the cache by its address.
2453    ///
2454    /// # Errors
2455    ///
2456    /// Returns an error if the pool is not registered in the cache.
2457    pub fn get_pool(&self, pool_identifier: &PoolIdentifier) -> anyhow::Result<&SharedPool> {
2458        match self.cache.get_pool(pool_identifier) {
2459            Some(pool) => Ok(pool),
2460            None => anyhow::bail!("Pool {pool_identifier} is not registered"),
2461        }
2462    }
2463
2464    /// Sends a data event to all subscribers through the data channel.
2465    pub fn send_data(&self, data: DataEvent) {
2466        if let Some(data_tx) = &self.data_tx {
2467            log::debug!("Sending {data}");
2468
2469            if let Err(e) = data_tx.send(data) {
2470                log::error!("Failed to send data: {e}");
2471            }
2472        } else {
2473            log::error!("No data event channel for sending data");
2474        }
2475    }
2476
2477    /// Disconnects all active connections and cleanup resources.
2478    ///
2479    /// This method should be called when shutting down the client to ensure
2480    /// proper cleanup of network connections and background tasks.
2481    pub async fn disconnect(&mut self) {
2482        self.subscription_manager.clear_block_demand();
2483        self.hypersync_client.disconnect().await;
2484    }
2485}
2486
2487#[cfg(test)]
2488mod tests {
2489    use std::str::FromStr;
2490
2491    use alloy::primitives::{Address, B256, I256, U160, U256, address};
2492    use nautilus_core::UnixNanos;
2493    use nautilus_model::defi::{
2494        Chain, Token,
2495        pool_analysis::{
2496            position::PoolPosition,
2497            snapshot::{PoolAnalytics, PoolState},
2498        },
2499    };
2500    use rstest::rstest;
2501    use tokio_util::sync::CancellationToken;
2502    use ustr::Ustr;
2503
2504    use super::*;
2505
2506    const WETH_USDT_CREATION_BLOCK: u64 = 12_375_326;
2507
2508    #[rstest]
2509    #[case(SnapshotValidation::OnChain, "on_chain", true)]
2510    #[case(SnapshotValidation::Replay, "replay", true)]
2511    #[case(SnapshotValidation::Invalid, "invalid", false)]
2512    fn snapshot_validation_db_token_and_usability(
2513        #[case] validation: SnapshotValidation,
2514        #[case] expected_str: &str,
2515        #[case] expected_usable: bool,
2516    ) {
2517        // as_str must match the pool_snapshot.validation_state CHECK values and the JSON contract;
2518        // is_usable must match the load filter `validation_state <> 'invalid'`.
2519        assert_eq!(validation.as_str(), expected_str);
2520        assert_eq!(validation.is_usable(), expected_usable);
2521        // from_db_token round-trips a stored token back to the enum, so a read-back verdict
2522        // reports the same state that was persisted.
2523        assert_eq!(
2524            SnapshotValidation::from_db_token(expected_str),
2525            Some(validation)
2526        );
2527    }
2528
2529    #[rstest]
2530    fn snapshot_validation_from_db_token_rejects_unknown() {
2531        assert_eq!(SnapshotValidation::from_db_token("bogus"), None);
2532    }
2533
2534    #[rstest]
2535    fn last_processed_event_for_on_chain_snapshot_rejects_unprocessed_profiler() {
2536        let mut profiler = PoolProfiler::new(weth_usdt_pool());
2537        profiler
2538            .initialize(U160::from_str_radix("3cb0adde486484998be0b", 16).unwrap())
2539            .expect("Known WETH/USDT initial sqrt price should initialize");
2540
2541        let error = BlockchainDataClientCore::last_processed_event_for_on_chain_snapshot(&profiler)
2542            .expect_err("unprocessed profiler should not fetch on-chain state");
2543
2544        assert_eq!(
2545            error.to_string(),
2546            format!(
2547                "cannot fetch on-chain snapshot for pool {} without a processed event",
2548                profiler.pool.address
2549            )
2550        );
2551    }
2552
2553    #[rstest]
2554    fn validate_rpc_snapshot_topology_accepts_consistent_snapshot() {
2555        let (profiler, snapshot) = rpc_topology_fixture();
2556
2557        let result = BlockchainDataClientCore::validate_rpc_snapshot_topology(&profiler, &snapshot);
2558
2559        assert!(result.is_ok());
2560    }
2561
2562    #[rstest]
2563    fn validate_rpc_snapshot_topology_rejects_missing_tick() {
2564        let (profiler, mut snapshot) = rpc_topology_fixture();
2565        snapshot.ticks.pop();
2566
2567        let error = BlockchainDataClientCore::validate_rpc_snapshot_topology(&profiler, &snapshot)
2568            .unwrap_err();
2569
2570        assert!(error.to_string().contains("expected 2 ticks, received 1"));
2571    }
2572
2573    #[rstest]
2574    fn validate_rpc_snapshot_topology_rejects_zeroed_position() {
2575        let (profiler, mut snapshot) = rpc_topology_fixture();
2576        snapshot.positions[0].liquidity = 0;
2577
2578        let error = BlockchainDataClientCore::validate_rpc_snapshot_topology(&profiler, &snapshot)
2579            .unwrap_err();
2580
2581        assert!(
2582            error
2583                .to_string()
2584                .contains("RPC positions do not match the complete HyperSync topology")
2585        );
2586    }
2587
2588    #[rstest]
2589    fn validate_rpc_snapshot_topology_rejects_active_liquidity_mismatch() {
2590        let (profiler, mut snapshot) = rpc_topology_fixture();
2591        snapshot.state.liquidity -= 1;
2592
2593        let error = BlockchainDataClientCore::validate_rpc_snapshot_topology(&profiler, &snapshot)
2594            .unwrap_err();
2595
2596        assert!(error.to_string().contains("RPC active liquidity mismatch"));
2597    }
2598
2599    #[rstest]
2600    fn validate_rpc_snapshot_topology_rejects_tick_liquidity_mismatch() {
2601        let (profiler, mut snapshot) = rpc_topology_fixture();
2602        snapshot.ticks[0].liquidity_gross -= 1;
2603
2604        let error = BlockchainDataClientCore::validate_rpc_snapshot_topology(&profiler, &snapshot)
2605            .unwrap_err();
2606
2607        assert!(error.to_string().contains("RPC tick -60 topology mismatch"));
2608    }
2609
2610    #[rstest]
2611    fn timestamp_for_on_chain_snapshot_prefers_cached_block_timestamp() {
2612        let pool = weth_usdt_pool();
2613        let mut profiler = PoolProfiler::new(pool);
2614        let cached_ts = UnixNanos::from(1_700_000_001_000_000_000);
2615        let profiler_ts = UnixNanos::from(1_700_000_000_000_000_000);
2616        profiler.last_processed_ts = Some(profiler_ts);
2617
2618        let timestamp =
2619            BlockchainDataClientCore::timestamp_for_on_chain_snapshot(&profiler, Some(cached_ts))
2620                .unwrap();
2621
2622        assert_eq!(timestamp, cached_ts);
2623    }
2624
2625    #[rstest]
2626    fn timestamp_for_on_chain_snapshot_falls_back_to_profiler_timestamp() {
2627        let pool = weth_usdt_pool();
2628        let mut profiler = PoolProfiler::new(pool);
2629        let profiler_ts = UnixNanos::from(1_700_000_000_000_000_000);
2630        profiler.last_processed_ts = Some(profiler_ts);
2631
2632        let timestamp =
2633            BlockchainDataClientCore::timestamp_for_on_chain_snapshot(&profiler, None).unwrap();
2634
2635        assert_eq!(timestamp, profiler_ts);
2636    }
2637
2638    #[rstest]
2639    fn timestamp_for_on_chain_snapshot_rejects_missing_timestamp() {
2640        let pool = weth_usdt_pool();
2641        let profiler = PoolProfiler::new(pool);
2642
2643        let error = BlockchainDataClientCore::timestamp_for_on_chain_snapshot(&profiler, None)
2644            .expect_err("missing timestamps should fail");
2645
2646        assert_eq!(
2647            error.to_string(),
2648            "missing block timestamp for on-chain snapshot"
2649        );
2650    }
2651
2652    #[rstest]
2653    fn block_scoped_snapshot_position_from_block_caches_timestamp_and_uses_sentinel_indexes() {
2654        let chain = Arc::new(
2655            Chain::from_chain_id(42161)
2656                .expect("Arbitrum chain should exist")
2657                .clone(),
2658        );
2659        let mut cache = BlockchainCache::new(chain);
2660        let timestamp = UnixNanos::from(1_700_000_002_000_000_000);
2661        let block = test_block(123, timestamp);
2662
2663        let position = BlockchainDataClientCore::block_scoped_snapshot_position_from_block(
2664            &mut cache, &block, 123,
2665        )
2666        .unwrap();
2667
2668        assert_eq!(position.number, 123);
2669        assert_eq!(position.transaction_hash, block.hash);
2670        assert_eq!(position.transaction_index, BLOCK_SCOPED_SNAPSHOT_INDEX);
2671        assert_eq!(position.log_index, BLOCK_SCOPED_SNAPSHOT_INDEX);
2672        assert_eq!(cache.get_block_timestamp(123), Some(&timestamp));
2673    }
2674
2675    #[rstest]
2676    fn block_scoped_snapshot_position_from_block_rejects_mismatched_block() {
2677        let chain = Arc::new(
2678            Chain::from_chain_id(42161)
2679                .expect("Arbitrum chain should exist")
2680                .clone(),
2681        );
2682        let mut cache = BlockchainCache::new(chain);
2683        let block = test_block(122, UnixNanos::from(1_700_000_002_000_000_000));
2684
2685        let error = BlockchainDataClientCore::block_scoped_snapshot_position_from_block(
2686            &mut cache, &block, 123,
2687        )
2688        .expect_err("mismatched block should fail");
2689
2690        assert_eq!(
2691            error.to_string(),
2692            "Fetched block 122 while requesting RPC snapshot block 123"
2693        );
2694        assert_eq!(cache.get_block_timestamp(122), None);
2695    }
2696
2697    #[rstest]
2698    #[case(100, 50, Some(99))]
2699    #[case(50, 50, None)]
2700    #[case(0, 0, None)]
2701    fn completed_pool_event_checkpoint_excludes_in_flight_block(
2702        #[case] block_number: u64,
2703        #[case] effective_from_block: u64,
2704        #[case] expected: Option<u64>,
2705    ) {
2706        let checkpoint = BlockchainDataClientCore::completed_pool_event_checkpoint(
2707            block_number,
2708            effective_from_block,
2709        );
2710
2711        assert_eq!(checkpoint, expected);
2712    }
2713
2714    #[rstest]
2715    fn pool_event_sync_ranges_preserve_legacy_history_during_new_family_backfill() {
2716        let families = vec![
2717            PoolEventFamily {
2718                name: "swap",
2719                signature: "swap".to_string(),
2720                introduced_version: 0,
2721            },
2722            PoolEventFamily {
2723                name: "fee_protocol_update",
2724                signature: "fee_protocol_update".to_string(),
2725                introduced_version: 1,
2726            },
2727            PoolEventFamily {
2728                name: "fee_protocol_collect",
2729                signature: "fee_protocol_collect".to_string(),
2730                introduced_version: 1,
2731            },
2732        ];
2733        let state = PoolEventSyncState {
2734            version: 0,
2735            last_full_sync_block: Some(100),
2736            family_blocks: Vec::new(),
2737        };
2738
2739        let ranges = BlockchainDataClientCore::pool_event_sync_ranges(&families, &state, 10, 110);
2740
2741        assert_eq!(
2742            ranges,
2743            vec![
2744                PoolEventSyncRange {
2745                    from_block: 10,
2746                    families: vec![families[1].clone(), families[2].clone()],
2747                },
2748                PoolEventSyncRange {
2749                    from_block: 101,
2750                    families: vec![families[0].clone()],
2751                },
2752            ]
2753        );
2754    }
2755
2756    #[rstest]
2757    fn pool_event_sync_ranges_group_families_by_checkpoint() {
2758        let families = vec![
2759            PoolEventFamily {
2760                name: "swap",
2761                signature: "swap".to_string(),
2762                introduced_version: 0,
2763            },
2764            PoolEventFamily {
2765                name: "fee_protocol_update",
2766                signature: "fee_protocol_update".to_string(),
2767                introduced_version: 1,
2768            },
2769            PoolEventFamily {
2770                name: "fee_protocol_collect",
2771                signature: "fee_protocol_collect".to_string(),
2772                introduced_version: 1,
2773            },
2774        ];
2775        let state = PoolEventSyncState {
2776            version: 1,
2777            last_full_sync_block: Some(100),
2778            family_blocks: vec![
2779                ("fee_protocol_collect".to_string(), 75),
2780                ("fee_protocol_update".to_string(), 50),
2781                ("swap".to_string(), 100),
2782            ],
2783        };
2784
2785        let ranges = BlockchainDataClientCore::pool_event_sync_ranges(&families, &state, 10, 100);
2786
2787        assert_eq!(
2788            ranges,
2789            vec![
2790                PoolEventSyncRange {
2791                    from_block: 51,
2792                    families: vec![families[1].clone()],
2793                },
2794                PoolEventSyncRange {
2795                    from_block: 76,
2796                    families: vec![families[2].clone()],
2797                },
2798            ]
2799        );
2800    }
2801
2802    #[rstest]
2803    fn pool_event_sync_ranges_backfill_future_family_without_schema_migration() {
2804        let families = vec![
2805            PoolEventFamily {
2806                name: "swap",
2807                signature: "swap".to_string(),
2808                introduced_version: 0,
2809            },
2810            PoolEventFamily {
2811                name: "future",
2812                signature: "future".to_string(),
2813                introduced_version: 2,
2814            },
2815        ];
2816        let state = PoolEventSyncState {
2817            version: 1,
2818            last_full_sync_block: Some(100),
2819            family_blocks: vec![("swap".to_string(), 100)],
2820        };
2821
2822        let ranges = BlockchainDataClientCore::pool_event_sync_ranges(&families, &state, 10, 100);
2823
2824        assert_eq!(
2825            ranges,
2826            vec![PoolEventSyncRange {
2827                from_block: 10,
2828                families: vec![families[1].clone()],
2829            }]
2830        );
2831    }
2832
2833    #[rstest]
2834    fn process_pool_swap_event_rejects_mismatched_block_hash() {
2835        let Some(mut core) = test_core().expect("test core builds") else {
2836            return;
2837        };
2838
2839        let pool = weth_usdt_pool();
2840        let block = test_block_with_hash(
2841            42,
2842            "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
2843            UnixNanos::from(1_700_000_000_000_000_000),
2844        );
2845        core.cache
2846            .cache_block_metadata(&block)
2847            .expect("test block hash must be valid");
2848
2849        let event = test_swap_event(
2850            &pool,
2851            42,
2852            Some("0xBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"),
2853        );
2854        let error = core
2855            .process_pool_swap_event(&event, &pool)
2856            .expect_err("mismatched observed hash should not convert");
2857
2858        assert_eq!(
2859            error.to_string(),
2860            "block hash mismatch for swap event at block 42: observed \
2861             0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb, cached \
2862             0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
2863        );
2864    }
2865
2866    #[rstest]
2867    fn process_pool_swap_event_attaches_matching_block_hash_and_timestamp() {
2868        let Some(mut core) = test_core().expect("test core builds") else {
2869            return;
2870        };
2871
2872        let pool = weth_usdt_pool();
2873        let timestamp = UnixNanos::from(1_700_000_000_000_000_001);
2874        let block = test_block_with_hash(
2875            42,
2876            "0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
2877            timestamp,
2878        );
2879        core.cache
2880            .cache_block_metadata(&block)
2881            .expect("test block hash must be valid");
2882
2883        let event = test_swap_event(
2884            &pool,
2885            42,
2886            Some("0xBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"),
2887        );
2888        let swap = core
2889            .process_pool_swap_event(&event, &pool)
2890            .expect("matching observed hash converts");
2891
2892        assert_eq!(swap.block, 42);
2893        assert_eq!(
2894            swap.block_hash,
2895            Some("0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb".to_string())
2896        );
2897        assert_eq!(swap.ts_event, timestamp);
2898        assert_eq!(swap.ts_init, timestamp);
2899    }
2900
2901    fn weth_usdt_pool() -> SharedPool {
2902        let chain = Arc::new(
2903            Chain::from_chain_id(1)
2904                .expect("Ethereum chain should exist")
2905                .clone(),
2906        );
2907        let dex = Arc::clone(
2908            &get_dex_extended(chain.name, &DexType::UniswapV3)
2909                .expect("Ethereum UniswapV3 should be registered")
2910                .dex,
2911        );
2912        let pool_address = address!("4e68ccd3e89f51c3074ca5072bbac773960dfa36");
2913        let token0 = Token::new(
2914            Arc::clone(&chain),
2915            address!("C02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2"),
2916            "Wrapped Ether".to_string(),
2917            "WETH".to_string(),
2918            18,
2919        );
2920        let token1 = Token::new(
2921            Arc::clone(&chain),
2922            address!("dAC17F958D2ee523a2206206994597C13D831ec7"),
2923            "Tether USD".to_string(),
2924            "USDT".to_string(),
2925            6,
2926        );
2927
2928        Arc::new(Pool::new(
2929            chain,
2930            dex,
2931            pool_address,
2932            PoolIdentifier::from_address(pool_address),
2933            WETH_USDT_CREATION_BLOCK,
2934            token0,
2935            token1,
2936            Some(3_000),
2937            Some(60),
2938            UnixNanos::default(),
2939        ))
2940    }
2941
2942    fn rpc_topology_fixture() -> (PoolProfiler, PoolSnapshot) {
2943        let pool = weth_usdt_pool();
2944        let owner = Address::ZERO;
2945        let liquidity = 1_000_u128;
2946        let state = PoolState {
2947            current_tick: 0,
2948            liquidity,
2949            ..Default::default()
2950        };
2951        let position = PoolPosition::new(owner, -60, 60, liquidity as i128);
2952        let ticks = vec![
2953            PoolTick::new(
2954                -60,
2955                liquidity,
2956                liquidity as i128,
2957                U256::ZERO,
2958                U256::ZERO,
2959                true,
2960                0,
2961            ),
2962            PoolTick::new(
2963                60,
2964                liquidity,
2965                -(liquidity as i128),
2966                U256::ZERO,
2967                U256::ZERO,
2968                true,
2969                0,
2970            ),
2971        ];
2972        let timestamp = UnixNanos::from(1_700_000_000_000_000_000);
2973        let snapshot = PoolSnapshot::new(
2974            pool.instrument_id,
2975            state,
2976            vec![position],
2977            ticks,
2978            PoolAnalytics::default(),
2979            BlockPosition::new(100, "0xabc".to_string(), 0, 0),
2980            timestamp,
2981            timestamp,
2982        );
2983        let mut profiler = PoolProfiler::new(pool);
2984        profiler.restore_from_snapshot(snapshot.clone()).unwrap();
2985
2986        (profiler, snapshot)
2987    }
2988
2989    fn test_block(number: u64, timestamp: UnixNanos) -> Block {
2990        Block::new(
2991            format!("0x{number:064x}"),
2992            String::from("0x0"),
2993            number,
2994            Ustr::from("0x0000000000000000000000000000000000000000"),
2995            30_000_000,
2996            21_000,
2997            timestamp,
2998            Some(Blockchain::Arbitrum),
2999        )
3000    }
3001
3002    fn test_block_with_hash(number: u64, hash: &str, timestamp: UnixNanos) -> Block {
3003        Block::new(
3004            hash.to_string(),
3005            String::from("0x0"),
3006            number,
3007            Ustr::from("0x0000000000000000000000000000000000000000"),
3008            30_000_000,
3009            21_000,
3010            timestamp,
3011            Some(Blockchain::Ethereum),
3012        )
3013    }
3014
3015    fn test_core() -> anyhow::Result<Option<BlockchainDataClientCore>> {
3016        if std::env::var("ENVIO_API_TOKEN").is_err() {
3017            return Ok(None);
3018        }
3019
3020        let chain = Arc::new(
3021            Chain::from_chain_id(1)
3022                .expect("Ethereum chain should exist")
3023                .clone(),
3024        );
3025        let config = BlockchainDataClientConfig::builder()
3026            .chain(Arc::clone(&chain))
3027            .dex_ids(vec![DexType::UniswapV3])
3028            .http_rpc_url("http://127.0.0.1:9".into())
3029            .build();
3030        let core = BlockchainDataClientCore::new(config, None, None, CancellationToken::new())?;
3031
3032        Ok(Some(core))
3033    }
3034
3035    fn fixture_hash(hash: &str) -> B256 {
3036        B256::from_str(hash).expect("fixture block hash must be valid")
3037    }
3038
3039    fn test_swap_event(
3040        pool: &SharedPool,
3041        block_number: u64,
3042        block_hash: Option<&str>,
3043    ) -> SwapEvent {
3044        let address = address!("1111111111111111111111111111111111111111");
3045
3046        SwapEvent::new(
3047            Arc::clone(&pool.dex),
3048            pool.pool_identifier,
3049            block_number,
3050            block_hash.map(fixture_hash),
3051            "0xdeadbeef".to_string(),
3052            0,
3053            7,
3054            address,
3055            address,
3056            I256::ZERO,
3057            I256::ZERO,
3058            U160::ZERO,
3059            0,
3060            0,
3061        )
3062    }
3063}