1use std::{cmp::max, collections::BTreeMap, sync::Arc};
17
18use ahash::AHashMap;
19use alloy::primitives::B256;
20use anyhow::Context;
21use futures_util::StreamExt;
22use nautilus_common::{live::sender::EventSender, messages::DataEvent};
23use nautilus_core::{
24 UnixNanos, hex,
25 string::{formatting::Separable, secret::REDACTED},
26};
27use nautilus_live::SocketControl;
28use nautilus_model::defi::{
29 Block, Blockchain, DexType, Pool, PoolIdentifier, PoolLiquidityUpdate, PoolProfiler, PoolSwap,
30 SharedChain, SharedDex, SharedPool,
31 data::{
32 DefiData, DexPoolData, PoolFeeCollect, PoolFeeProtocolCollect, PoolFeeProtocolUpdate,
33 PoolFlash,
34 block::{BLOCK_SCOPED_SNAPSHOT_INDEX, BlockPosition},
35 },
36 pool_analysis::{compare::compare_pool_profiler_detailed, snapshot::PoolSnapshot},
37 reporting::{BlockchainSyncReportItems, BlockchainSyncReporter},
38 tick_map::tick::PoolTick,
39};
40use nautilus_network::websocket::TransportBackend;
41
42use crate::{
43 cache::{BlockchainCache, PoolEventSyncState},
44 config::BlockchainDataClientConfig,
45 contracts::{
46 erc20::Erc20Contract,
47 uniswap_v3_pool::{FeeProtocolEncoding, UniswapV3PoolContract},
48 },
49 data::{block_hash_matches, subscription::DefiDataSubscriptionManager},
50 events::{
51 burn::BurnEvent, collect::CollectEvent, fee_protocol_collect::FeeProtocolCollectEvent,
52 fee_protocol_update::FeeProtocolUpdateEvent, flash::FlashEvent, mint::MintEvent,
53 swap::SwapEvent,
54 },
55 exchanges::{extended::DexExtended, get_dex_extended},
56 hypersync::{
57 client::{HyperSyncClient, PoolEventStreamItem},
58 log::{extract_block_number, extract_event_signature_bytes},
59 },
60 rpc::{
61 BlockchainRpcClient, BlockchainRpcClientAny,
62 chains::{
63 arbitrum::ArbitrumRpcClient, base::BaseRpcClient, bsc::BscRpcClient,
64 ethereum::EthereumRpcClient, polygon::PolygonRpcClient,
65 },
66 http::BlockchainHttpRpcClient,
67 types::BlockchainMessage,
68 },
69 services::PoolDiscoveryService,
70};
71
72const BLOCKS_PROCESS_IN_SYNC_REPORT: u64 = 50_000;
73const POOL_EVENT_BLOCK_BATCH_SIZE: usize = 20_000;
74const POOL_EVENT_SYNC_VERSION_LEGACY: u32 = 0;
75const POOL_EVENT_SYNC_VERSION_PROTOCOL_FEE: u32 = 1;
76const POOL_EVENT_SYNC_VERSION: u32 = POOL_EVENT_SYNC_VERSION_PROTOCOL_FEE;
78#[derive(Debug, Clone, PartialEq, Eq)]
81struct PoolEventFamily {
82 name: &'static str,
83 signature: String,
84 introduced_version: u32,
85}
86
87#[derive(Debug, Clone, PartialEq, Eq)]
88struct PoolEventSyncRange {
89 from_block: u64,
90 families: Vec<PoolEventFamily>,
91}
92
93#[derive(Debug)]
98pub struct BlockchainDataClientCore {
99 pub chain: SharedChain,
101 pub config: BlockchainDataClientConfig,
103 pub cache: BlockchainCache,
105 tokens: Erc20Contract,
107 univ3_pool: UniswapV3PoolContract,
109 pub hypersync_client: HyperSyncClient,
111 pub rpc_client: Option<BlockchainRpcClientAny>,
113 pub subscription_manager: DefiDataSubscriptionManager,
115 data_tx: Option<EventSender<DataEvent>>,
117 cancellation_token: tokio_util::sync::CancellationToken,
119}
120
121#[derive(Debug, Clone, Copy, PartialEq, Eq)]
123pub enum SnapshotValidation {
124 OnChain,
126 Replay,
129 Invalid,
131}
132
133impl SnapshotValidation {
134 #[must_use]
136 pub const fn is_usable(self) -> bool {
137 !matches!(self, Self::Invalid)
138 }
139
140 #[must_use]
142 pub const fn as_str(self) -> &'static str {
143 match self {
144 Self::OnChain => "on_chain",
145 Self::Replay => "replay",
146 Self::Invalid => "invalid",
147 }
148 }
149
150 #[must_use]
154 pub fn from_db_token(token: &str) -> Option<Self> {
155 match token {
156 "on_chain" => Some(Self::OnChain),
157 "replay" => Some(Self::Replay),
158 "invalid" => Some(Self::Invalid),
159 _ => None,
160 }
161 }
162}
163
164impl BlockchainDataClientCore {
165 pub fn new(
171 config: BlockchainDataClientConfig,
172 hypersync_tx: Option<tokio::sync::mpsc::UnboundedSender<BlockchainMessage>>,
173 data_tx: Option<EventSender<DataEvent>>,
174 cancellation_token: tokio_util::sync::CancellationToken,
175 ) -> anyhow::Result<Self> {
176 config.validate()?;
177
178 let chain = Arc::clone(&config.chain);
179 let cache = BlockchainCache::new(Arc::clone(&chain));
180
181 log::debug!(
183 "Initializing blockchain data client for '{}' with HTTP RPC: {}",
184 chain.name,
185 REDACTED
186 );
187
188 let rpc_client = if let (false, Some(wss_rpc_url)) = (
189 config.use_hypersync_for_live_data,
190 config.wss_rpc_url.as_ref(),
191 ) {
192 log::debug!("WebSocket RPC URL: {REDACTED}");
193 Some(Self::initialize_rpc_client(
194 chain.name,
195 wss_rpc_url.clone().into_inner(),
196 config.transport_backend,
197 config.proxy_url.clone().map(|value| value.into_inner()),
198 ))
199 } else {
200 log::debug!("Using HyperSync for live data (no WebSocket RPC)");
201 None
202 };
203 let http_rpc_client = Arc::new(BlockchainHttpRpcClient::new(
204 config.http_rpc_url.clone().into_inner(),
205 config.rpc_requests_per_second,
206 config.proxy_url.clone().map(|value| value.into_inner()),
207 )?);
208 let multicall_calls_per_rpc_request = config.multicall_calls_per_rpc_request;
209 let erc20_contract = Erc20Contract::new(
210 Arc::clone(&http_rpc_client),
211 config.pool_filters.remove_pools_with_empty_erc20fields,
212 );
213
214 let hypersync_client =
215 HyperSyncClient::new(Arc::clone(&chain), hypersync_tx, cancellation_token.clone())?;
216 Ok(Self {
217 chain,
218 config,
219 rpc_client,
220 tokens: erc20_contract,
221 univ3_pool: UniswapV3PoolContract::new(
222 http_rpc_client,
223 multicall_calls_per_rpc_request,
224 ),
225 cache,
226 hypersync_client,
227 subscription_manager: DefiDataSubscriptionManager::new(),
228 data_tx,
229 cancellation_token,
230 })
231 }
232
233 pub fn set_socket_control(&mut self, control: SocketControl) {
235 if let Some(rpc_client) = &mut self.rpc_client {
236 rpc_client.set_socket_control(control);
237 }
238 }
239
240 pub async fn initialize_cache_database(&mut self) {
242 if let Some(pg_connect_options) = &self.config.postgres_cache_database_config {
243 log::debug!(
244 "Initializing blockchain cache on database '{}'",
245 pg_connect_options.database
246 );
247 self.cache
248 .initialize_database(pg_connect_options.clone().into())
249 .await;
250 }
251 }
252
253 fn initialize_rpc_client(
255 blockchain: Blockchain,
256 wss_rpc_url: String,
257 transport_backend: TransportBackend,
258 proxy_url: Option<String>,
259 ) -> BlockchainRpcClientAny {
260 let mut client = match blockchain {
261 Blockchain::Ethereum => {
262 BlockchainRpcClientAny::Ethereum(EthereumRpcClient::new(wss_rpc_url, proxy_url))
263 }
264 Blockchain::Polygon => {
265 BlockchainRpcClientAny::Polygon(PolygonRpcClient::new(wss_rpc_url, proxy_url))
266 }
267 Blockchain::Base => {
268 BlockchainRpcClientAny::Base(BaseRpcClient::new(wss_rpc_url, proxy_url))
269 }
270 Blockchain::Arbitrum => {
271 BlockchainRpcClientAny::Arbitrum(ArbitrumRpcClient::new(wss_rpc_url, proxy_url))
272 }
273 Blockchain::Bsc => {
274 BlockchainRpcClientAny::Bsc(BscRpcClient::new(wss_rpc_url, proxy_url))
275 }
276 _ => panic!("Unsupported blockchain {blockchain} for RPC connection"),
277 };
278 client.set_transport_backend(transport_backend);
279 client
280 }
281
282 pub async fn connect(&mut self) -> anyhow::Result<()> {
289 self.config.validate()?;
290
291 log::debug!(
292 "Connecting blockchain data client for '{}'",
293 self.chain.name
294 );
295 self.initialize_cache_database().await;
296
297 if let Some(ref mut rpc_client) = self.rpc_client {
298 rpc_client.connect().await?;
299 }
300
301 let from_block = self.determine_from_block();
302
303 log::debug!(
304 "Connecting to blockchain data source for '{}' from block {}",
305 self.chain.name,
306 from_block.separate_with_commas()
307 );
308
309 self.cache.initialize_chain().await;
311 self.cache.connect(from_block).await?;
313 for dex in self.config.dex_ids.clone() {
317 self.register_dex_exchange(dex).await?;
318 self.sync_exchange_pools(&dex, from_block, None, false)
319 .await?;
320 }
321
322 Ok(())
323 }
324
325 pub async fn sync_blocks_checked(
331 &mut self,
332 from_block: u64,
333 to_block: Option<u64>,
334 ) -> anyhow::Result<()> {
335 if let Some(blocks_status) = self.cache.get_cache_block_consistency_status().await {
336 if blocks_status.is_consistent() {
338 log::debug!(
339 "Cache is consistent: no gaps detected (last continuous block: {})",
340 blocks_status.last_continuous_block
341 );
342 let target_block = max(blocks_status.max_block + 1, from_block);
343 log::debug!(
344 "Starting fast sync with COPY from block {}",
345 target_block.separate_with_commas()
346 );
347 self.sync_blocks(target_block, to_block, true).await?;
348 } else {
349 let gap_size = blocks_status.max_block - blocks_status.last_continuous_block;
350 log::debug!(
351 "Cache inconsistency detected: {} blocks missing between {} and {}",
352 gap_size,
353 blocks_status.last_continuous_block + 1,
354 blocks_status.max_block
355 );
356
357 log::debug!(
358 "Block syncing Phase 1: Filling gaps with INSERT (blocks {} to {})",
359 blocks_status.last_continuous_block + 1,
360 blocks_status.max_block
361 );
362 self.sync_blocks(
363 blocks_status.last_continuous_block + 1,
364 Some(blocks_status.max_block),
365 false,
366 )
367 .await?;
368
369 log::debug!(
370 "Block syncing Phase 2: Continuing with fast COPY from block {}",
371 (blocks_status.max_block + 1).separate_with_commas()
372 );
373 self.sync_blocks(blocks_status.max_block + 1, to_block, true)
374 .await?;
375 }
376 } else {
377 self.sync_blocks(from_block, to_block, true).await?;
378 }
379
380 Ok(())
381 }
382
383 pub async fn sync_blocks(
389 &mut self,
390 from_block: u64,
391 to_block: Option<u64>,
392 use_copy_command: bool,
393 ) -> anyhow::Result<()> {
394 const BATCH_SIZE: usize = 1000;
395
396 let to_block = if let Some(block) = to_block {
397 block
398 } else {
399 self.hypersync_client.current_block().await
400 };
401 let total_blocks = to_block.saturating_sub(from_block) + 1;
402 log::debug!(
403 "Syncing blocks from {} to {} (total: {} blocks)",
404 from_block.separate_with_commas(),
405 to_block.separate_with_commas(),
406 total_blocks.separate_with_commas()
407 );
408
409 if let Err(e) = self.cache.toggle_performance_settings(true).await {
411 log::warn!("Failed to enable performance settings: {e}");
412 }
413
414 let blocks_stream = self
415 .hypersync_client
416 .request_blocks_stream(from_block, Some(to_block))
417 .await;
418
419 tokio::pin!(blocks_stream);
420
421 let mut metrics = BlockchainSyncReporter::new(
422 BlockchainSyncReportItems::Blocks,
423 from_block,
424 total_blocks,
425 BLOCKS_PROCESS_IN_SYNC_REPORT,
426 );
427
428 let mut batch: Vec<Block> = Vec::with_capacity(BATCH_SIZE);
429
430 let cancellation_token = self.cancellation_token.clone();
431 let sync_result = tokio::select! {
432 () = cancellation_token.cancelled() => {
433 log::debug!("Block sync cancelled");
434 Err(anyhow::anyhow!("Sync cancelled"))
435 }
436 result = async {
437 while let Some(block) = blocks_stream.next().await {
438 let block_number = block.number;
439 if self.cache.get_block_timestamp(block_number).is_some() {
440 continue;
441 }
442 batch.push(block);
443
444 if batch.len() >= BATCH_SIZE || block_number >= to_block {
446 let batch_size = batch.len();
447
448 self.cache.add_blocks_batch(batch, use_copy_command).await?;
449 metrics.update(batch_size);
450
451 batch = Vec::with_capacity(BATCH_SIZE);
453 }
454
455 if metrics.should_log_progress(block_number, to_block) {
457 metrics.log_progress(block_number);
458 }
459 }
460
461 if !batch.is_empty() {
463 let batch_size = batch.len();
464 self.cache.add_blocks_batch(batch, use_copy_command).await?;
465 metrics.update(batch_size);
466 }
467
468 metrics.log_final_stats();
469 Ok(())
470 } => result
471 };
472
473 sync_result?;
474
475 if let Err(e) = self.cache.toggle_performance_settings(false).await {
477 log::warn!("Failed to restore default settings: {e}");
478 }
479
480 Ok(())
481 }
482
483 pub async fn sync_pool_events(
489 &mut self,
490 dex: &DexType,
491 pool_identifier: PoolIdentifier,
492 from_block: Option<u64>,
493 to_block: Option<u64>,
494 reset: bool,
495 ) -> anyhow::Result<()> {
496 const EVENT_BATCH_SIZE: usize = 20000;
497
498 let pool: SharedPool = Arc::clone(self.get_pool(&pool_identifier)?);
499 let pool_display = pool.to_full_spec_string();
500 let from_block = from_block.unwrap_or(pool.creation_block);
501 let pool_address = &pool.address;
503
504 let dex_extended = self.get_dex_extended(dex)?.clone();
505 let event_families = Self::pool_event_families(&dex_extended);
506 let sync_state = if reset {
507 PoolEventSyncState::default()
508 } else {
509 self.cache
510 .get_pool_event_sync_state(dex, &pool_identifier)
511 .await?
512 };
513
514 let to_block = match to_block {
515 Some(block) => block,
516 None => self.hypersync_client.current_block().await,
517 };
518
519 if from_block > to_block {
520 log::debug!(
521 "D {} Pool '{}' requested event range {} to {} is empty, skipping sync",
522 dex,
523 pool_display,
524 from_block.separate_with_commas(),
525 to_block.separate_with_commas(),
526 );
527 return Ok(());
528 }
529
530 let sync_ranges =
531 Self::pool_event_sync_ranges(&event_families, &sync_state, from_block, to_block);
532 let inherited_event_family_names = event_families
533 .iter()
534 .filter(|family| {
535 sync_state.version == POOL_EVENT_SYNC_VERSION_LEGACY
536 && family.introduced_version == POOL_EVENT_SYNC_VERSION_LEGACY
537 && !sync_state
538 .family_blocks
539 .iter()
540 .any(|(name, _)| name == family.name)
541 })
542 .map(|family| family.name)
543 .collect::<Vec<_>>();
544
545 let Some(sync_range) = sync_ranges.first().cloned() else {
546 if sync_state.version < POOL_EVENT_SYNC_VERSION {
547 if let Some(last_full_sync_block) = sync_state.last_full_sync_block {
548 self.cache
549 .update_pool_event_sync(
550 dex,
551 &pool_identifier,
552 &inherited_event_family_names,
553 last_full_sync_block,
554 None,
555 )
556 .await?;
557 }
558 self.cache
559 .update_pool_event_sync(
560 dex,
561 &pool_identifier,
562 &[],
563 to_block,
564 Some(POOL_EVENT_SYNC_VERSION),
565 )
566 .await?;
567 }
568
569 log::debug!(
570 "D {} Pool '{}' event families already synced to block {}, skipping sync",
571 dex,
572 pool_display,
573 to_block.separate_with_commas()
574 );
575 return Ok(());
576 };
577 let has_more_ranges = sync_ranges.len() > 1;
578
579 let last_block_across_pool_events_table = self
581 .cache
582 .get_pool_event_tables_last_block(&pool_identifier)
583 .await?;
584
585 let effective_from_block = sync_range.from_block;
586 let event_family_names = sync_range
587 .families
588 .iter()
589 .map(|family| family.name)
590 .collect::<Vec<_>>();
591 let total_blocks = to_block.saturating_sub(effective_from_block) + 1;
592 log::debug!(
593 "Syncing Pool: '{}' event families {:?} from {} to {} (total: {} blocks)",
594 pool_display,
595 event_family_names,
596 effective_from_block.separate_with_commas(),
597 to_block.separate_with_commas(),
598 total_blocks.separate_with_commas(),
599 );
600
601 let mut metrics = BlockchainSyncReporter::new(
602 BlockchainSyncReportItems::PoolEvents,
603 effective_from_block,
604 total_blocks,
605 BLOCKS_PROCESS_IN_SYNC_REPORT,
606 );
607 let swap_event_signature = dex_extended.swap_created_event.as_ref();
608 let mint_event_signature = dex_extended.mint_created_event.as_ref();
609 let burn_event_signature = dex_extended.burn_created_event.as_ref();
610 let collect_event_signature = dex_extended.collect_created_event.as_ref();
611 let flash_event_signature = dex_extended.flash_created_event.as_ref();
612 let protocol_update_event_signature = dex_extended.fee_protocol_update_event.as_ref();
613 let protocol_collect_event_signature = dex_extended.fee_protocol_collect_event.as_ref();
614 let initialize_event_signature: Option<&str> =
615 dex_extended.initialize_event.as_ref().map(|s| s.as_ref());
616
617 let swap_sig_bytes = hex::decode(
619 swap_event_signature
620 .strip_prefix("0x")
621 .unwrap_or(swap_event_signature),
622 )?;
623 let mint_sig_bytes = hex::decode(
624 mint_event_signature
625 .strip_prefix("0x")
626 .unwrap_or(mint_event_signature),
627 )?;
628 let burn_sig_bytes = hex::decode(
629 burn_event_signature
630 .strip_prefix("0x")
631 .unwrap_or(burn_event_signature),
632 )?;
633 let collect_sig_bytes = hex::decode(
634 collect_event_signature
635 .strip_prefix("0x")
636 .unwrap_or(collect_event_signature),
637 )?;
638 let flash_sig_bytes = flash_event_signature
639 .map(|s| hex::decode(s.strip_prefix("0x").unwrap_or(s)).unwrap_or_default());
640 let protocol_update_sig_bytes = protocol_update_event_signature
641 .map(|s| hex::decode(s.strip_prefix("0x").unwrap_or(s)).unwrap_or_default());
642 let protocol_collect_sig_bytes = protocol_collect_event_signature
643 .map(|s| hex::decode(s.strip_prefix("0x").unwrap_or(s)).unwrap_or_default());
644 let initialize_sig_bytes = initialize_event_signature
645 .map(|s| hex::decode(s.strip_prefix("0x").unwrap_or(s)).unwrap_or_default());
646
647 let event_signatures = sync_range
648 .families
649 .iter()
650 .map(|family| family.signature.as_str())
651 .collect();
652
653 let pool_events_stream = self
654 .hypersync_client
655 .request_contract_events_stream(
656 effective_from_block,
657 Some(to_block),
658 pool_address,
659 event_signatures,
660 )
661 .await;
662 tokio::pin!(pool_events_stream);
663
664 let mut last_block_saved = effective_from_block;
665 let mut blocks_processed = 0;
666
667 let mut block_batch: Vec<Block> = Vec::with_capacity(POOL_EVENT_BLOCK_BATCH_SIZE);
668 let mut swap_batch: Vec<PoolSwap> = Vec::with_capacity(EVENT_BATCH_SIZE);
669 let mut liquidity_batch: Vec<PoolLiquidityUpdate> = Vec::with_capacity(EVENT_BATCH_SIZE);
670 let mut collect_batch: Vec<PoolFeeCollect> = Vec::with_capacity(EVENT_BATCH_SIZE);
671 let mut protocol_update_batch: Vec<PoolFeeProtocolUpdate> =
672 Vec::with_capacity(EVENT_BATCH_SIZE);
673 let mut protocol_collect_batch: Vec<PoolFeeProtocolCollect> =
674 Vec::with_capacity(EVENT_BATCH_SIZE);
675 let mut flash_batch: Vec<PoolFlash> = Vec::with_capacity(EVENT_BATCH_SIZE);
676
677 let mut beyond_stale_data = last_block_across_pool_events_table
679 .is_none_or(|tables_max| effective_from_block > tables_max);
680
681 let cancellation_token = self.cancellation_token.clone();
682 let sync_result = tokio::select! {
683 () = cancellation_token.cancelled() => {
684 log::debug!("Pool event sync cancelled");
685 Err(anyhow::anyhow!("Sync cancelled"))
686 }
687 result = async {
688 while let Some(item) = pool_events_stream.next().await {
689 let log = match item {
690 PoolEventStreamItem::Block(block) => {
691 self.record_pool_event_block(block, &mut block_batch).await?;
692 continue;
693 }
694 PoolEventStreamItem::Log(log) => log,
695 };
696 let block_number = extract_block_number(&log)?;
697 blocks_processed += block_number - last_block_saved;
698 last_block_saved = block_number;
699
700 let event_sig_bytes = extract_event_signature_bytes(&log)?;
701 if event_sig_bytes == swap_sig_bytes.as_slice() {
702 let swap_event = dex_extended.parse_swap_event_hypersync(&log)?;
703 let swap = self
704 .process_pool_swap_event(&swap_event, &pool)
705 .with_context(|| {
706 format!("failed to process swap event at block {}", swap_event.block_number)
707 })?;
708 swap_batch.push(swap);
709 } else if event_sig_bytes == mint_sig_bytes.as_slice() {
710 let mint_event = dex_extended.parse_mint_event_hypersync(&log)?;
711 let liquidity_update = self
712 .process_pool_mint_event(&mint_event, &pool, &dex_extended)
713 .with_context(|| {
714 format!("failed to process mint event at block {}", mint_event.block_number)
715 })?;
716 liquidity_batch.push(liquidity_update);
717 } else if event_sig_bytes == burn_sig_bytes.as_slice() {
718 let burn_event = dex_extended.parse_burn_event_hypersync(&log)?;
719 let liquidity_update = self
720 .process_pool_burn_event(&burn_event, &pool, &dex_extended)
721 .with_context(|| {
722 format!("failed to process burn event at block {}", burn_event.block_number)
723 })?;
724 liquidity_batch.push(liquidity_update);
725 } else if event_sig_bytes == collect_sig_bytes.as_slice() {
726 let collect_event = dex_extended.parse_collect_event_hypersync(&log)?;
727 let fee_collect = self
728 .process_pool_collect_event(&collect_event, &pool, &dex_extended)
729 .with_context(|| {
730 format!(
731 "failed to process collect event at block {}",
732 collect_event.block_number
733 )
734 })?;
735 collect_batch.push(fee_collect);
736 } else if initialize_sig_bytes.as_ref().is_some_and(|sig| sig.as_slice() == event_sig_bytes) {
737 let initialize_event = dex_extended.parse_initialize_event_hypersync(&log)?;
738 self.cache
739 .update_pool_initialize_price_tick(&initialize_event)
740 .await?;
741 } else if protocol_update_sig_bytes.as_ref().is_some_and(|sig| sig.as_slice() == event_sig_bytes) {
742 let fee_protocol_update_event = dex_extended.parse_fee_protocol_update_event_hypersync(&log)?;
743 let update = self
744 .process_pool_fee_protocol_update_event(&fee_protocol_update_event, &pool)
745 .with_context(|| {
746 format!(
747 "failed to process SetFeeProtocol event at block {}",
748 fee_protocol_update_event.block_number
749 )
750 })?;
751 protocol_update_batch.push(update);
752 } else if protocol_collect_sig_bytes.as_ref().is_some_and(|sig| sig.as_slice() == event_sig_bytes) {
753 let fee_protocol_collect_event = dex_extended.parse_fee_protocol_collect_event_hypersync(&log)?;
754 let collect = self
755 .process_pool_fee_protocol_collect_event(&fee_protocol_collect_event, &pool)
756 .with_context(|| {
757 format!(
758 "failed to process CollectProtocol event at block {}",
759 fee_protocol_collect_event.block_number
760 )
761 })?;
762 protocol_collect_batch.push(collect);
763 } else if flash_sig_bytes.as_ref().is_some_and(|sig| sig.as_slice() == event_sig_bytes) {
764 let parse_fn = dex_extended
765 .parse_flash_event_hypersync_fn
766 .context("missing flash event parser")?;
767 let flash_event = parse_fn(Arc::clone(&dex_extended.dex), &log)
768 .context("failed to parse flash event")?;
769 let flash = self
770 .process_pool_flash_event(&flash_event, &pool)
771 .with_context(|| {
772 format!("failed to process flash event at block {}", flash_event.block_number)
773 })?;
774 flash_batch.push(flash);
775 } else {
776 let event_signature = hex::encode(event_sig_bytes);
777 anyhow::bail!("unexpected event signature {event_signature} for log {log:?}");
778 }
779
780 if !beyond_stale_data
782 && last_block_across_pool_events_table
783 .is_some_and(|table_max| block_number > table_max)
784 {
785 log::debug!(
786 "Crossed beyond stale data at block {block_number} - flushing current batches with ON CONFLICT, then switching to COPY"
787 );
788
789 self.flush_event_batches(
791 EVENT_BATCH_SIZE,
792 &mut block_batch,
793 &mut swap_batch,
794 &mut liquidity_batch,
795 &mut collect_batch,
796 &mut protocol_update_batch,
797 &mut protocol_collect_batch,
798 &mut flash_batch,
799 false,
800 true,
801 )
802 .await?;
803
804 beyond_stale_data = true;
805 log::debug!("Switched to COPY mode - future batches will use COPY command");
806 } else {
807 self.flush_event_batches(
809 EVENT_BATCH_SIZE,
810 &mut block_batch,
811 &mut swap_batch,
812 &mut liquidity_batch,
813 &mut collect_batch,
814 &mut protocol_update_batch,
815 &mut protocol_collect_batch,
816 &mut flash_batch,
817 false, false,
819 )
820 .await?;
821 }
822
823 metrics.update(blocks_processed as usize);
824 blocks_processed = 0;
825
826 if metrics.should_log_progress(block_number, to_block) {
828 metrics.log_progress(block_number);
829 self.flush_event_batches(
830 EVENT_BATCH_SIZE,
831 &mut block_batch,
832 &mut swap_batch,
833 &mut liquidity_batch,
834 &mut collect_batch,
835 &mut protocol_update_batch,
836 &mut protocol_collect_batch,
837 &mut flash_batch,
838 false,
839 true,
840 )
841 .await?;
842
843 if let Some(checkpoint_block) =
844 Self::completed_pool_event_checkpoint(block_number, effective_from_block)
845 {
846 self.cache
847 .update_pool_event_sync(
848 dex,
849 &pool_identifier,
850 &event_family_names,
851 checkpoint_block,
852 None,
853 )
854 .await?;
855 }
856 }
857 }
858
859 self.flush_event_batches(
860 EVENT_BATCH_SIZE,
861 &mut block_batch,
862 &mut swap_batch,
863 &mut liquidity_batch,
864 &mut collect_batch,
865 &mut protocol_update_batch,
866 &mut protocol_collect_batch,
867 &mut flash_batch,
868 false,
869 true,
870 )
871 .await?;
872
873 metrics.log_final_stats();
874 self.cache
875 .update_pool_event_sync(
876 dex,
877 &pool_identifier,
878 &event_family_names,
879 to_block,
880 None,
881 )
882 .await?;
883
884 log::debug!(
885 "Successfully synced Dex '{}' Pool '{}' event families {:?} up to block {}",
886 dex,
887 pool_display,
888 event_family_names,
889 to_block.separate_with_commas()
890 );
891 Ok(())
892 } => result
893 };
894
895 sync_result?;
896
897 if has_more_ranges {
898 return Box::pin(self.sync_pool_events(
899 dex,
900 pool_identifier,
901 Some(from_block),
902 Some(to_block),
903 false,
904 ))
905 .await;
906 }
907
908 if let Some(last_full_sync_block) = sync_state.last_full_sync_block {
909 self.cache
910 .update_pool_event_sync(
911 dex,
912 &pool_identifier,
913 &inherited_event_family_names,
914 last_full_sync_block,
915 None,
916 )
917 .await?;
918 }
919 self.cache
920 .update_pool_event_sync(
921 dex,
922 &pool_identifier,
923 &[],
924 to_block,
925 Some(POOL_EVENT_SYNC_VERSION),
926 )
927 .await?;
928
929 Ok(())
930 }
931
932 fn pool_event_families(dex: &DexExtended) -> Vec<PoolEventFamily> {
933 let mut families = vec![
934 PoolEventFamily {
935 name: "swap",
936 signature: dex.swap_created_event.to_string(),
937 introduced_version: POOL_EVENT_SYNC_VERSION_LEGACY,
938 },
939 PoolEventFamily {
940 name: "mint",
941 signature: dex.mint_created_event.to_string(),
942 introduced_version: POOL_EVENT_SYNC_VERSION_LEGACY,
943 },
944 PoolEventFamily {
945 name: "burn",
946 signature: dex.burn_created_event.to_string(),
947 introduced_version: POOL_EVENT_SYNC_VERSION_LEGACY,
948 },
949 PoolEventFamily {
950 name: "collect",
951 signature: dex.collect_created_event.to_string(),
952 introduced_version: POOL_EVENT_SYNC_VERSION_LEGACY,
953 },
954 ];
955
956 if let Some(signature) = &dex.initialize_event {
957 families.push(PoolEventFamily {
958 name: "initialize",
959 signature: signature.to_string(),
960 introduced_version: POOL_EVENT_SYNC_VERSION_LEGACY,
961 });
962 }
963
964 if let Some(signature) = &dex.flash_created_event {
965 families.push(PoolEventFamily {
966 name: "flash",
967 signature: signature.to_string(),
968 introduced_version: POOL_EVENT_SYNC_VERSION_LEGACY,
969 });
970 }
971
972 if let Some(signature) = &dex.fee_protocol_update_event {
973 families.push(PoolEventFamily {
974 name: "fee_protocol_update",
975 signature: signature.to_string(),
976 introduced_version: POOL_EVENT_SYNC_VERSION_PROTOCOL_FEE,
977 });
978 }
979
980 if let Some(signature) = &dex.fee_protocol_collect_event {
981 families.push(PoolEventFamily {
982 name: "fee_protocol_collect",
983 signature: signature.to_string(),
984 introduced_version: POOL_EVENT_SYNC_VERSION_PROTOCOL_FEE,
985 });
986 }
987
988 families
989 }
990
991 fn pool_event_sync_ranges(
992 families: &[PoolEventFamily],
993 state: &PoolEventSyncState,
994 from_block: u64,
995 to_block: u64,
996 ) -> Vec<PoolEventSyncRange> {
997 let mut grouped = BTreeMap::<u64, Vec<PoolEventFamily>>::new();
998
999 for family in families {
1000 let explicit_checkpoint = state
1001 .family_blocks
1002 .iter()
1003 .find_map(|(name, block)| (name == family.name).then_some(*block));
1004 let legacy_checkpoint = (state.version == POOL_EVENT_SYNC_VERSION_LEGACY
1005 && family.introduced_version == POOL_EVENT_SYNC_VERSION_LEGACY)
1006 .then_some(state.last_full_sync_block)
1007 .flatten();
1008 let family_from_block = explicit_checkpoint
1009 .or(legacy_checkpoint)
1010 .map_or(from_block, |block| max(from_block, block.saturating_add(1)));
1011
1012 if family_from_block <= to_block {
1013 grouped
1014 .entry(family_from_block)
1015 .or_default()
1016 .push(family.clone());
1017 }
1018 }
1019
1020 grouped
1021 .into_iter()
1022 .map(|(from_block, families)| PoolEventSyncRange {
1023 from_block,
1024 families,
1025 })
1026 .collect()
1027 }
1028
1029 #[expect(clippy::too_many_arguments)]
1030 async fn flush_event_batches(
1031 &mut self,
1032 event_batch_size: usize,
1033 block_batch: &mut Vec<Block>,
1034 swap_batch: &mut Vec<PoolSwap>,
1035 liquidity_batch: &mut Vec<PoolLiquidityUpdate>,
1036 collect_batch: &mut Vec<PoolFeeCollect>,
1037 protocol_update_batch: &mut Vec<PoolFeeProtocolUpdate>,
1038 protocol_collect_batch: &mut Vec<PoolFeeProtocolCollect>,
1039 flash_batch: &mut Vec<PoolFlash>,
1040 use_copy_command: bool,
1041 force_flush_all: bool,
1042 ) -> anyhow::Result<()> {
1043 let should_flush_swaps =
1044 (force_flush_all || swap_batch.len() >= event_batch_size) && !swap_batch.is_empty();
1045 let should_flush_liquidity = (force_flush_all || liquidity_batch.len() >= event_batch_size)
1046 && !liquidity_batch.is_empty();
1047 let should_flush_collects = (force_flush_all || collect_batch.len() >= event_batch_size)
1048 && !collect_batch.is_empty();
1049 let should_flush_protocol_update = (force_flush_all
1050 || protocol_update_batch.len() >= event_batch_size)
1051 && !protocol_update_batch.is_empty();
1052 let should_flush_protocol_collect = (force_flush_all
1053 || protocol_collect_batch.len() >= event_batch_size)
1054 && !protocol_collect_batch.is_empty();
1055 let should_flush_flash =
1056 (force_flush_all || flash_batch.len() >= event_batch_size) && !flash_batch.is_empty();
1057
1058 if force_flush_all
1059 || should_flush_swaps
1060 || should_flush_liquidity
1061 || should_flush_collects
1062 || should_flush_protocol_update
1063 || should_flush_protocol_collect
1064 || should_flush_flash
1065 {
1066 self.flush_pool_event_blocks(block_batch).await?;
1067 }
1068
1069 if should_flush_swaps {
1070 self.cache
1071 .add_pool_swaps_batch(swap_batch, use_copy_command)
1072 .await?;
1073 swap_batch.clear();
1074 }
1075
1076 if should_flush_liquidity {
1077 self.cache
1078 .add_pool_liquidity_updates_batch(liquidity_batch, use_copy_command)
1079 .await?;
1080 liquidity_batch.clear();
1081 }
1082
1083 if should_flush_collects {
1084 self.cache
1085 .add_pool_fee_collects_batch(collect_batch, use_copy_command)
1086 .await?;
1087 collect_batch.clear();
1088 }
1089
1090 if should_flush_protocol_update {
1091 self.cache
1092 .add_pool_fee_protocol_updates_batch(protocol_update_batch)
1093 .await?;
1094 protocol_update_batch.clear();
1095 }
1096
1097 if should_flush_protocol_collect {
1098 self.cache
1099 .add_pool_fee_protocol_collect_batch(protocol_collect_batch)
1100 .await?;
1101 protocol_collect_batch.clear();
1102 }
1103
1104 if should_flush_flash {
1105 self.cache.add_pool_flash_batch(flash_batch).await?;
1106 flash_batch.clear();
1107 }
1108 Ok(())
1109 }
1110
1111 async fn record_pool_event_block(
1112 &mut self,
1113 block: Block,
1114 block_batch: &mut Vec<Block>,
1115 ) -> anyhow::Result<()> {
1116 self.cache.cache_block_metadata(&block)?;
1117 block_batch.push(block);
1118 if block_batch.len() >= POOL_EVENT_BLOCK_BATCH_SIZE {
1119 self.flush_pool_event_blocks(block_batch).await?;
1120 }
1121 Ok(())
1122 }
1123
1124 async fn flush_pool_event_blocks(
1125 &mut self,
1126 block_batch: &mut Vec<Block>,
1127 ) -> anyhow::Result<()> {
1128 if block_batch.is_empty() {
1129 return Ok(());
1130 }
1131
1132 self.cache
1133 .add_pool_event_blocks_batch(std::mem::take(block_batch))
1134 .await
1135 }
1136
1137 fn completed_pool_event_checkpoint(
1138 block_number: u64,
1139 effective_from_block: u64,
1140 ) -> Option<u64> {
1141 let checkpoint_block = block_number.checked_sub(1)?;
1142 (checkpoint_block >= effective_from_block).then_some(checkpoint_block)
1143 }
1144
1145 pub fn process_pool_swap_event(
1157 &self,
1158 swap_event: &SwapEvent,
1159 pool: &SharedPool,
1160 ) -> anyhow::Result<PoolSwap> {
1161 let timestamp = self
1162 .cache
1163 .get_block_timestamp(swap_event.block_number)
1164 .copied()
1165 .context("missing block timestamp for swap event")?;
1166 let mut swap = swap_event.to_pool_swap(
1167 Arc::clone(&self.chain),
1168 pool.instrument_id,
1169 pool.pool_identifier,
1170 timestamp,
1171 );
1172 swap.block_hash = Some(self.observed_block_hash(
1173 swap_event.block_number,
1174 swap_event.block_hash,
1175 "swap",
1176 )?);
1177
1178 if let Err(e) = swap.calculate_trade_info(&pool.token0, &pool.token1, None) {
1180 log::debug!(
1181 "Skipping trade info for swap at block {} on pool {}: {e}",
1182 swap_event.block_number,
1183 pool.instrument_id,
1184 );
1185 }
1186
1187 Ok(swap)
1188 }
1189
1190 pub fn process_pool_mint_event(
1196 &self,
1197 mint_event: &MintEvent,
1198 pool: &SharedPool,
1199 dex_extended: &DexExtended,
1200 ) -> anyhow::Result<PoolLiquidityUpdate> {
1201 let timestamp = self
1202 .cache
1203 .get_block_timestamp(mint_event.block_number)
1204 .copied()
1205 .context("missing block timestamp for mint event")?;
1206
1207 let mut liquidity_update = mint_event.to_pool_liquidity_update(
1208 Arc::clone(&self.chain),
1209 Arc::clone(&dex_extended.dex),
1210 pool.instrument_id,
1211 timestamp,
1212 );
1213 liquidity_update.block_hash = Some(self.observed_block_hash(
1214 mint_event.block_number,
1215 mint_event.block_hash,
1216 "mint",
1217 )?);
1218
1219 Ok(liquidity_update)
1222 }
1223
1224 pub fn process_pool_burn_event(
1231 &self,
1232 burn_event: &BurnEvent,
1233 pool: &SharedPool,
1234 dex_extended: &DexExtended,
1235 ) -> anyhow::Result<PoolLiquidityUpdate> {
1236 let timestamp = self
1237 .cache
1238 .get_block_timestamp(burn_event.block_number)
1239 .copied()
1240 .context("missing block timestamp for burn event")?;
1241
1242 let mut liquidity_update = burn_event.to_pool_liquidity_update(
1243 Arc::clone(&self.chain),
1244 Arc::clone(&dex_extended.dex),
1245 pool.instrument_id,
1246 pool.pool_identifier,
1247 timestamp,
1248 );
1249 liquidity_update.block_hash = Some(self.observed_block_hash(
1250 burn_event.block_number,
1251 burn_event.block_hash,
1252 "burn",
1253 )?);
1254
1255 Ok(liquidity_update)
1258 }
1259
1260 pub fn process_pool_collect_event(
1266 &self,
1267 collect_event: &CollectEvent,
1268 pool: &SharedPool,
1269 dex_extended: &DexExtended,
1270 ) -> anyhow::Result<PoolFeeCollect> {
1271 let timestamp = self
1272 .cache
1273 .get_block_timestamp(collect_event.block_number)
1274 .copied()
1275 .context("missing block timestamp for collect event")?;
1276
1277 let mut fee_collect = collect_event.to_pool_fee_collect(
1278 Arc::clone(&self.chain),
1279 Arc::clone(&dex_extended.dex),
1280 pool.instrument_id,
1281 timestamp,
1282 );
1283 fee_collect.block_hash = Some(self.observed_block_hash(
1284 collect_event.block_number,
1285 collect_event.block_hash,
1286 "collect",
1287 )?);
1288
1289 Ok(fee_collect)
1290 }
1291
1292 pub fn process_pool_flash_event(
1298 &self,
1299 flash_event: &FlashEvent,
1300 pool: &SharedPool,
1301 ) -> anyhow::Result<PoolFlash> {
1302 let timestamp = self
1303 .cache
1304 .get_block_timestamp(flash_event.block_number)
1305 .copied()
1306 .context("missing block timestamp for flash event")?;
1307
1308 let mut flash =
1309 flash_event.to_pool_flash(Arc::clone(&self.chain), pool.instrument_id, timestamp);
1310 flash.block_hash = Some(self.observed_block_hash(
1311 flash_event.block_number,
1312 flash_event.block_hash,
1313 "flash",
1314 )?);
1315
1316 Ok(flash)
1317 }
1318
1319 pub fn process_pool_fee_protocol_update_event(
1327 &self,
1328 fee_protocol_update_event: &FeeProtocolUpdateEvent,
1329 pool: &SharedPool,
1330 ) -> anyhow::Result<PoolFeeProtocolUpdate> {
1331 let timestamp = self
1332 .cache
1333 .get_block_timestamp(fee_protocol_update_event.block_number)
1334 .copied()
1335 .context("missing block timestamp for SetFeeProtocol event")?;
1336
1337 let mut update = fee_protocol_update_event.to_pool_fee_protocol_update(
1338 Arc::clone(&self.chain),
1339 pool.instrument_id,
1340 timestamp,
1341 );
1342 update.block_hash = Some(self.observed_block_hash(
1343 fee_protocol_update_event.block_number,
1344 fee_protocol_update_event.block_hash,
1345 "SetFeeProtocol",
1346 )?);
1347
1348 Ok(update)
1349 }
1350
1351 pub fn process_pool_fee_protocol_collect_event(
1359 &self,
1360 fee_protocol_collect_event: &FeeProtocolCollectEvent,
1361 pool: &SharedPool,
1362 ) -> anyhow::Result<PoolFeeProtocolCollect> {
1363 let timestamp = self
1364 .cache
1365 .get_block_timestamp(fee_protocol_collect_event.block_number)
1366 .copied()
1367 .context("missing block timestamp for CollectProtocol event")?;
1368
1369 let mut collect = fee_protocol_collect_event.to_pool_fee_protocol_collect(
1370 Arc::clone(&self.chain),
1371 pool.instrument_id,
1372 timestamp,
1373 );
1374 collect.block_hash = Some(self.observed_block_hash(
1375 fee_protocol_collect_event.block_number,
1376 fee_protocol_collect_event.block_hash,
1377 "CollectProtocol",
1378 )?);
1379
1380 Ok(collect)
1381 }
1382
1383 fn observed_block_hash(
1384 &self,
1385 block_number: u64,
1386 observed_hash: Option<B256>,
1387 event: &str,
1388 ) -> anyhow::Result<String> {
1389 let cached_hash = self.cache.get_block_hash(block_number).with_context(|| {
1390 format!("missing block hash for {event} event at block {block_number}")
1391 })?;
1392
1393 if !block_hash_matches(observed_hash, cached_hash) {
1394 anyhow::bail!(
1395 "block hash mismatch for {event} event at block {block_number}: observed {}, cached {cached_hash}",
1396 observed_hash.map_or_else(|| "none".to_string(), |hash| hash.to_string())
1397 );
1398 }
1399
1400 Ok(cached_hash.to_string())
1401 }
1402
1403 pub async fn sync_exchange_pools(
1414 &mut self,
1415 dex: &DexType,
1416 from_block: u64,
1417 to_block: Option<u64>,
1418 reset: bool,
1419 ) -> anyhow::Result<()> {
1420 let dex_extended = self.get_dex_extended(dex)?.clone();
1421
1422 let mut service = PoolDiscoveryService::new(
1423 Arc::clone(&self.chain),
1424 &mut self.cache,
1425 &self.tokens,
1426 &self.hypersync_client,
1427 self.cancellation_token.clone(),
1428 self.config.clone(),
1429 );
1430
1431 service
1432 .sync_pools(&dex_extended, from_block, to_block, reset)
1433 .await?;
1434
1435 Ok(())
1436 }
1437
1438 pub async fn register_dex_exchange(&mut self, dex_id: DexType) -> anyhow::Result<()> {
1449 self.register_dex(dex_id).await?;
1450 let _ = self.cache.load_pools(&dex_id).await?;
1451 Ok(())
1452 }
1453
1454 pub async fn register_dex_exchange_for_pool(
1464 &mut self,
1465 dex_id: DexType,
1466 pool_identifier: &PoolIdentifier,
1467 ) -> anyhow::Result<()> {
1468 self.register_dex(dex_id).await?;
1469 let _ = self.cache.load_pool(&dex_id, pool_identifier).await?;
1470 Ok(())
1471 }
1472
1473 async fn register_dex(&mut self, dex_id: DexType) -> anyhow::Result<()> {
1475 let Some(dex_extended) = get_dex_extended(self.chain.name, &dex_id) else {
1476 anyhow::bail!("Unknown DEX {dex_id} on chain {}", self.chain.name);
1477 };
1478
1479 log::debug!("Registering DEX {dex_id} on chain {}", self.chain.name);
1480 self.cache.add_dex(Arc::clone(&dex_extended.dex)).await?;
1481 self.subscription_manager.register_dex_for_subscriptions(
1482 dex_id,
1483 dex_extended.swap_created_event.as_ref(),
1484 dex_extended.mint_created_event.as_ref(),
1485 dex_extended.burn_created_event.as_ref(),
1486 dex_extended.collect_created_event.as_ref(),
1487 dex_extended.flash_created_event.as_deref(),
1488 );
1489 self.subscription_manager.register_dex_fee_protocol_events(
1490 dex_id,
1491 dex_extended.fee_protocol_update_event.as_deref(),
1492 dex_extended.fee_protocol_collect_event.as_deref(),
1493 );
1494 Ok(())
1495 }
1496
1497 pub async fn bootstrap_latest_pool_profiler(
1518 &mut self,
1519 pool: &SharedPool,
1520 to_block: Option<u64>,
1521 ) -> anyhow::Result<(PoolProfiler, bool)> {
1522 log::debug!(
1523 "Bootstrapping latest pool profiler for pool {}",
1524 pool.address
1525 );
1526
1527 if self.cache.database.is_none() {
1528 anyhow::bail!(
1529 "Database is not initialized, so we cannot properly bootstrap the latest pool profiler"
1530 );
1531 }
1532
1533 let to_block = match to_block {
1534 Some(block) => block,
1535 None => self.hypersync_client.current_block().await,
1536 };
1537 let (mut profiler, from_position) = self
1538 .seed_pool_profiler_from_latest_snapshot(pool, to_block)
1539 .await?;
1540
1541 if self
1545 .cache
1546 .database
1547 .as_ref()
1548 .unwrap()
1549 .get_pool_last_synced_block(self.chain.chain_id, &pool.dex.name, &pool.pool_identifier)
1550 .await?
1551 .is_none()
1552 {
1553 return self
1554 .construct_pool_profiler_from_hypersync_rpc(profiler, from_position, to_block)
1555 .await;
1556 }
1557
1558 self.sync_pool_events(
1560 &pool.dex.name,
1561 pool.pool_identifier,
1562 None,
1563 Some(to_block),
1564 false,
1565 )
1566 .await
1567 .context("failed to sync pool events for snapshot request")?;
1568
1569 if !profiler.is_initialized {
1570 if let Some(initial_sqrt_price_x96) = pool.initial_sqrt_price_x96 {
1571 profiler.initialize(initial_sqrt_price_x96)?;
1572 } else {
1573 anyhow::bail!(
1574 "Pool is not initialized and it doesn't contain initial price, cannot bootstrap profiler"
1575 );
1576 }
1577 }
1578
1579 let from_block = from_position
1580 .as_ref()
1581 .map_or(profiler.pool.creation_block, |block_position| {
1582 block_position.number
1583 });
1584 let total_blocks = to_block.saturating_sub(from_block) + 1;
1585
1586 profiler.enable_reporting(from_block, total_blocks, BLOCKS_PROCESS_IN_SYNC_REPORT);
1588
1589 let mut stream = self.cache.database.as_ref().unwrap().stream_pool_events(
1590 Arc::clone(&pool.chain),
1591 Arc::clone(&pool.dex),
1592 pool.instrument_id,
1593 pool.pool_identifier,
1594 from_position.clone(),
1595 Some(to_block),
1596 );
1597
1598 while let Some(result) = stream.next().await {
1599 match result {
1600 Ok(event) => {
1601 profiler.process(&event)?;
1602 }
1603 Err(e) => return Err(e).context("failed to stream pool event from database"),
1604 }
1605 }
1606
1607 profiler.finalize_reporting();
1608
1609 Ok((profiler, false))
1610 }
1611
1612 pub async fn bootstrap_pool_profiler_from_rpc_snapshot(
1621 &mut self,
1622 pool: &SharedPool,
1623 to_block: u64,
1624 ) -> anyhow::Result<(PoolProfiler, bool)> {
1625 if self.cache.database.is_none() {
1626 anyhow::bail!(
1627 "Database is not initialized, so we cannot bootstrap the pool profiler from an RPC snapshot"
1628 );
1629 }
1630
1631 self.construct_pool_profiler_from_hypersync_rpc(
1632 PoolProfiler::new(Arc::clone(pool)),
1633 None,
1634 to_block,
1635 )
1636 .await
1637 }
1638
1639 pub async fn advance_pool_profiler_from_rpc_snapshot(
1650 &mut self,
1651 profiler: PoolProfiler,
1652 to_block: u64,
1653 ) -> anyhow::Result<(PoolProfiler, bool)> {
1654 let from_position = profiler.last_processed_event.clone().ok_or_else(|| {
1655 anyhow::anyhow!("cannot advance an RPC profiler without a snapshot watermark")
1656 })?;
1657
1658 if to_block < from_position.number {
1659 anyhow::bail!(
1660 "cannot advance RPC profiler from block {} to earlier block {to_block}",
1661 from_position.number
1662 );
1663 }
1664
1665 self.construct_pool_profiler_from_hypersync_rpc(profiler, Some(from_position), to_block)
1666 .await
1667 }
1668
1669 async fn seed_pool_profiler_from_latest_snapshot(
1670 &self,
1671 pool: &SharedPool,
1672 to_block: u64,
1673 ) -> anyhow::Result<(PoolProfiler, Option<BlockPosition>)> {
1674 let mut profiler = PoolProfiler::new(Arc::clone(pool));
1675
1676 let from_position = match self
1677 .cache
1678 .database
1679 .as_ref()
1680 .expect("database presence is checked by caller")
1681 .load_latest_pool_snapshot(
1682 pool.chain.chain_id,
1683 &pool.pool_identifier,
1684 Some(to_block),
1685 true,
1686 )
1687 .await
1688 {
1689 Ok(Some(snapshot)) => {
1690 if snapshot.positions.is_empty()
1696 && snapshot.ticks.is_empty()
1697 && snapshot.block_position.number == pool.creation_block
1698 {
1699 log::warn!(
1700 "Ignoring empty stub snapshot at pool creation block {} for {}; rebuilding from events",
1701 snapshot.block_position.number.separate_with_commas(),
1702 pool.instrument_id,
1703 );
1704 None
1705 } else {
1706 log::debug!(
1707 "Loaded valid snapshot from block {} which contains {} positions and {} ticks",
1708 snapshot.block_position.number.separate_with_commas(),
1709 snapshot.positions.len(),
1710 snapshot.ticks.len()
1711 );
1712 let block_position = snapshot.block_position.clone();
1713 profiler.restore_from_snapshot(snapshot)?;
1714 log::debug!("Restored profiler from snapshot");
1715 Some(block_position)
1716 }
1717 }
1718 _ => {
1719 log::debug!("No valid snapshot found, processing from beginning");
1720 None
1721 }
1722 };
1723
1724 Ok((profiler, from_position))
1725 }
1726
1727 async fn construct_pool_profiler_from_hypersync_rpc(
1748 &mut self,
1749 mut profiler: PoolProfiler,
1750 from_position: Option<BlockPosition>,
1751 to_block: u64,
1752 ) -> anyhow::Result<(PoolProfiler, bool)> {
1753 log::debug!("Constructing pool profiler from hypersync stream and RPC target block state");
1754 let dex_extended = self.get_dex_extended(&profiler.pool.dex.name)?.clone();
1755 let mint_event_signature = dex_extended.mint_created_event.as_ref();
1756 let burn_event_signature = dex_extended.burn_created_event.as_ref();
1757 let initialize_event_signature =
1758 if let Some(initialize_event) = &dex_extended.initialize_event {
1759 initialize_event.as_ref()
1760 } else {
1761 anyhow::bail!(
1762 "DEX {} does not have initialize event set.",
1763 profiler.pool.dex.name
1764 );
1765 };
1766 let mint_sig_bytes = hex::decode(
1767 mint_event_signature
1768 .strip_prefix("0x")
1769 .unwrap_or(mint_event_signature),
1770 )?;
1771 let burn_sig_bytes = hex::decode(
1772 burn_event_signature
1773 .strip_prefix("0x")
1774 .unwrap_or(burn_event_signature),
1775 )?;
1776 let initialize_sig_bytes = hex::decode(
1777 initialize_event_signature
1778 .strip_prefix("0x")
1779 .unwrap_or(initialize_event_signature),
1780 )?;
1781 let protocol_update_event_signature = dex_extended.fee_protocol_update_event.as_deref();
1782 let protocol_update_sig_bytes = protocol_update_event_signature
1783 .map(|s| hex::decode(s.strip_prefix("0x").unwrap_or(s)).unwrap_or_default());
1784 let protocol_collect_event_signature = dex_extended.fee_protocol_collect_event.as_deref();
1785 let protocol_collect_sig_bytes = protocol_collect_event_signature
1786 .map(|s| hex::decode(s.strip_prefix("0x").unwrap_or(s)).unwrap_or_default());
1787
1788 let from_block = from_position.map_or(profiler.pool.creation_block, |block_position| {
1789 block_position.number
1790 });
1791 let total_blocks = to_block.saturating_sub(from_block) + 1;
1792
1793 log::debug!(
1794 "Bootstrapping pool profiler for pool {} from block {} to {} (total: {} blocks)",
1795 profiler.pool.address,
1796 from_block.separate_with_commas(),
1797 to_block.separate_with_commas(),
1798 total_blocks.separate_with_commas()
1799 );
1800
1801 profiler.enable_reporting(from_block, total_blocks, BLOCKS_PROCESS_IN_SYNC_REPORT);
1803
1804 let mut event_signatures = vec![
1805 mint_event_signature,
1806 burn_event_signature,
1807 initialize_event_signature,
1808 ];
1809
1810 if let Some(event) = protocol_update_event_signature {
1811 event_signatures.push(event);
1812 }
1813
1814 if let Some(event) = protocol_collect_event_signature {
1815 event_signatures.push(event);
1816 }
1817
1818 let pool_events_stream = self
1819 .hypersync_client
1820 .request_contract_events_stream(
1821 from_block,
1822 Some(to_block),
1823 &profiler.pool.address,
1824 event_signatures,
1825 )
1826 .await;
1827 tokio::pin!(pool_events_stream);
1828 let mut block_batch: Vec<Block> = Vec::with_capacity(POOL_EVENT_BLOCK_BATCH_SIZE);
1829
1830 while let Some(item) = pool_events_stream.next().await {
1831 let log = match item {
1832 PoolEventStreamItem::Block(block) => {
1833 self.record_pool_event_block(block, &mut block_batch)
1834 .await?;
1835 continue;
1836 }
1837 PoolEventStreamItem::Log(log) => log,
1838 };
1839 let event_sig_bytes = extract_event_signature_bytes(&log)?;
1840
1841 if event_sig_bytes == initialize_sig_bytes {
1842 if profiler.is_initialized {
1843 log::debug!(
1847 "Profiler already initialized; skipping Initialize event at block {}",
1848 extract_block_number(&log)?.separate_with_commas(),
1849 );
1850 } else {
1851 let initialize_event = dex_extended.parse_initialize_event_hypersync(&log)?;
1852 profiler.initialize(initialize_event.sqrt_price_x96)?;
1853 self.cache
1854 .database
1855 .as_ref()
1856 .unwrap()
1857 .update_pool_initial_price_tick(self.chain.chain_id, &initialize_event)
1858 .await?;
1859 }
1860 } else if event_sig_bytes == mint_sig_bytes {
1861 let mint_event = dex_extended.parse_mint_event_hypersync(&log)?;
1862 let liquidity_update = self
1863 .process_pool_mint_event(&mint_event, &profiler.pool, &dex_extended)
1864 .with_context(|| {
1865 format!(
1866 "failed to process mint event at block {}",
1867 mint_event.block_number
1868 )
1869 })?;
1870 profiler.process(&DexPoolData::LiquidityUpdate(liquidity_update))?;
1871 } else if event_sig_bytes == burn_sig_bytes {
1872 let burn_event = dex_extended.parse_burn_event_hypersync(&log)?;
1873 let liquidity_update = self
1874 .process_pool_burn_event(&burn_event, &profiler.pool, &dex_extended)
1875 .with_context(|| {
1876 format!(
1877 "failed to process burn event at block {}",
1878 burn_event.block_number
1879 )
1880 })?;
1881 profiler.process(&DexPoolData::LiquidityUpdate(liquidity_update))?;
1882 } else if protocol_update_sig_bytes
1883 .as_ref()
1884 .is_some_and(|sig| sig.as_slice() == event_sig_bytes)
1885 {
1886 let fee_protocol_update_event =
1887 dex_extended.parse_fee_protocol_update_event_hypersync(&log)?;
1888 let update = self
1889 .process_pool_fee_protocol_update_event(
1890 &fee_protocol_update_event,
1891 &profiler.pool,
1892 )
1893 .with_context(|| {
1894 format!(
1895 "failed to process SetFeeProtocol event at block {}",
1896 fee_protocol_update_event.block_number
1897 )
1898 })?;
1899 profiler.process(&DexPoolData::FeeProtocolUpdate(update))?;
1900 } else if protocol_collect_sig_bytes
1901 .as_ref()
1902 .is_some_and(|sig| sig.as_slice() == event_sig_bytes)
1903 {
1904 let fee_protocol_collect_event =
1905 dex_extended.parse_fee_protocol_collect_event_hypersync(&log)?;
1906 let collect = self
1907 .process_pool_fee_protocol_collect_event(
1908 &fee_protocol_collect_event,
1909 &profiler.pool,
1910 )
1911 .with_context(|| {
1912 format!(
1913 "failed to process CollectProtocol event at block {}",
1914 fee_protocol_collect_event.block_number
1915 )
1916 })?;
1917 profiler.process(&DexPoolData::FeeProtocolCollect(collect))?;
1918 } else {
1919 let event_signature = hex::encode(event_sig_bytes);
1920 anyhow::bail!(
1921 "unexpected event signature in bootstrap_latest_pool_profiler: {event_signature} for log {log:?}"
1922 );
1923 }
1924 }
1925
1926 self.flush_pool_event_blocks(&mut block_batch).await?;
1927 profiler.finalize_reporting();
1928
1929 let snapshot_block_position = self.block_scoped_snapshot_position(to_block).await?;
1930 let on_chain_snapshot = self
1931 .get_on_chain_snapshot_at_position(&profiler, snapshot_block_position)
1932 .await
1933 .with_context(|| {
1934 format!(
1935 "failed to restore pool {} from RPC snapshot at target block {} with {} ticks and {} positions",
1936 profiler.pool.address,
1937 to_block.separate_with_commas(),
1938 profiler.get_active_tick_values().len().separate_with_commas(),
1939 profiler.get_all_position_keys().len().separate_with_commas()
1940 )
1941 })?;
1942 Self::validate_rpc_snapshot_topology(&profiler, &on_chain_snapshot).with_context(|| {
1943 format!(
1944 "RPC snapshot topology validation failed for pool {} at block {}",
1945 profiler.pool.address,
1946 to_block.separate_with_commas()
1947 )
1948 })?;
1949 profiler.restore_from_snapshot(on_chain_snapshot)?;
1950
1951 Ok((profiler, true))
1952 }
1953
1954 pub async fn check_snapshot_validity(
1974 &self,
1975 profiler: &PoolProfiler,
1976 already_validated: bool,
1977 ) -> anyhow::Result<SnapshotValidation> {
1978 let (validation, block_position) = if already_validated {
1979 log::debug!("Snapshot already validated from RPC, skipping on-chain comparison");
1981 let last_event = profiler
1982 .last_processed_event
1983 .clone()
1984 .expect("Profiler should have last_processed_event");
1985 (SnapshotValidation::OnChain, Some(last_event))
1986 } else {
1987 match self.get_on_chain_snapshot(profiler).await {
1989 Ok(on_chain_snapshot) => {
1990 log::debug!("Comparing profiler state with on-chain state...");
1991 let comparison = compare_pool_profiler_detailed(profiler, &on_chain_snapshot);
1992 let validation = if comparison.is_valid_for_snapshot() {
1993 if !comparison.is_exact_match() {
1994 log::warn!(
1995 "Pool profiler snapshot has a non-structural mismatch (sqrt ratio, fee protocol, or protocol fees); accepting snapshot"
1996 );
1997 }
1998 SnapshotValidation::OnChain
1999 } else {
2000 log::error!(
2001 "Pool profiler state does NOT match on-chain smart contract state"
2002 );
2003 SnapshotValidation::Invalid
2004 };
2005 (validation, Some(on_chain_snapshot.block_position))
2006 }
2007 Err(e) => {
2008 log::warn!(
2009 "Could not validate snapshot against on-chain state, keeping replay-derived snapshot: {e}"
2010 );
2011 let reported = self
2015 .stored_snapshot_validation(profiler)
2016 .await?
2017 .unwrap_or(SnapshotValidation::Replay);
2018 (reported, None)
2019 }
2020 }
2021 };
2022
2023 if let (Some(block_position), Some(cache_database)) = (block_position, &self.cache.database)
2024 {
2025 cache_database
2026 .set_pool_snapshot_validation_state(
2027 profiler.pool.chain.chain_id,
2028 &profiler.pool.pool_identifier,
2029 block_position.number,
2030 block_position.transaction_index,
2031 block_position.log_index,
2032 validation.as_str(),
2033 )
2034 .await?;
2035 log::debug!(
2036 "Set pool snapshot validation state to {}",
2037 validation.as_str()
2038 );
2039 }
2040
2041 Ok(validation)
2042 }
2043
2044 async fn stored_snapshot_validation(
2049 &self,
2050 profiler: &PoolProfiler,
2051 ) -> anyhow::Result<Option<SnapshotValidation>> {
2052 let (Some(block_position), Some(cache_database)) =
2053 (profiler.last_processed_event.as_ref(), &self.cache.database)
2054 else {
2055 return Ok(None);
2056 };
2057
2058 let stored = cache_database
2059 .get_pool_snapshot_validation_state(
2060 profiler.pool.chain.chain_id,
2061 &profiler.pool.pool_identifier,
2062 block_position.number,
2063 block_position.transaction_index,
2064 block_position.log_index,
2065 )
2066 .await?;
2067
2068 Ok(stored.and_then(|token| SnapshotValidation::from_db_token(&token)))
2069 }
2070
2071 async fn get_on_chain_snapshot(&self, profiler: &PoolProfiler) -> anyhow::Result<PoolSnapshot> {
2077 let last_processed_event = Self::last_processed_event_for_on_chain_snapshot(profiler)?;
2078 self.get_on_chain_snapshot_at_position(profiler, last_processed_event)
2079 .await
2080 }
2081
2082 async fn get_on_chain_snapshot_at_position(
2083 &self,
2084 profiler: &PoolProfiler,
2085 block_position: BlockPosition,
2086 ) -> anyhow::Result<PoolSnapshot> {
2087 if matches!(
2089 profiler.pool.dex.name,
2090 DexType::UniswapV3 | DexType::PancakeSwapV3
2091 ) {
2092 let fee_protocol_encoding = match profiler.pool.dex.name {
2093 DexType::PancakeSwapV3 => FeeProtocolEncoding::PancakeSwapV3BasisPoints,
2094 _ => FeeProtocolEncoding::UniswapV3Packed,
2095 };
2096 let timestamp = Self::timestamp_for_on_chain_snapshot(
2097 profiler,
2098 self.cache
2099 .get_block_timestamp(block_position.number)
2100 .copied(),
2101 )?;
2102 let on_chain_snapshot = self
2103 .univ3_pool
2104 .fetch_snapshot(
2105 &profiler.pool.address,
2106 profiler.pool.instrument_id,
2107 profiler.get_active_tick_values().as_slice(),
2108 &profiler.get_all_position_keys(),
2109 block_position,
2110 timestamp, timestamp, fee_protocol_encoding,
2113 )
2114 .await?;
2115
2116 Ok(on_chain_snapshot)
2117 } else {
2118 anyhow::bail!(
2119 "Fetching on-chain snapshot for Dex protocol {} is not supported yet.",
2120 profiler.pool.dex.name
2121 )
2122 }
2123 }
2124
2125 fn validate_rpc_snapshot_topology(
2126 profiler: &PoolProfiler,
2127 snapshot: &PoolSnapshot,
2128 ) -> anyhow::Result<()> {
2129 let tick_spacing = i32::try_from(
2130 profiler
2131 .pool
2132 .tick_spacing
2133 .context("pool tick spacing is not set")?,
2134 )?;
2135 let expected_positions: AHashMap<_, _> = profiler
2136 .get_all_positions()
2137 .into_iter()
2138 .map(|position| {
2139 (
2140 (position.owner, position.tick_lower, position.tick_upper),
2141 position.liquidity,
2142 )
2143 })
2144 .collect();
2145 let actual_positions: AHashMap<_, _> = snapshot
2146 .positions
2147 .iter()
2148 .map(|position| {
2149 (
2150 (position.owner, position.tick_lower, position.tick_upper),
2151 position.liquidity,
2152 )
2153 })
2154 .collect();
2155
2156 if actual_positions.len() != snapshot.positions.len()
2157 || actual_positions != expected_positions
2158 {
2159 anyhow::bail!(
2160 "RPC positions do not match the complete HyperSync topology: expected {} positions, received {}",
2161 expected_positions.len(),
2162 actual_positions.len()
2163 );
2164 }
2165
2166 let actual_ticks: AHashMap<_, _> = snapshot
2167 .ticks
2168 .iter()
2169 .map(|tick| (tick.value, tick))
2170 .collect();
2171
2172 if actual_ticks.len() != snapshot.ticks.len() {
2173 anyhow::bail!("RPC snapshot contains duplicate ticks");
2174 }
2175
2176 let expected_tick_values = profiler.get_active_tick_values();
2177 if actual_ticks.len() != expected_tick_values.len() {
2178 anyhow::bail!(
2179 "RPC ticks do not match the complete HyperSync topology: expected {} ticks, received {}",
2180 expected_tick_values.len(),
2181 actual_ticks.len()
2182 );
2183 }
2184
2185 for tick_value in expected_tick_values {
2186 let expected_tick = profiler
2187 .get_tick(tick_value)
2188 .with_context(|| format!("missing replay tick {tick_value}"))?;
2189 let actual_tick = actual_ticks
2190 .get(&tick_value)
2191 .with_context(|| format!("RPC snapshot omitted tick {tick_value}"))?;
2192
2193 if !actual_tick.initialized
2194 || actual_tick.liquidity_gross == 0
2195 || actual_tick.liquidity_gross != expected_tick.liquidity_gross
2196 || actual_tick.liquidity_net != expected_tick.liquidity_net
2197 {
2198 anyhow::bail!(
2199 "RPC tick {tick_value} topology mismatch: expected gross={} net={}, received gross={} net={} initialized={}",
2200 expected_tick.liquidity_gross,
2201 expected_tick.liquidity_net,
2202 actual_tick.liquidity_gross,
2203 actual_tick.liquidity_net,
2204 actual_tick.initialized
2205 );
2206 }
2207 }
2208
2209 let mut derived_ticks: AHashMap<i32, (u128, i128)> = AHashMap::new();
2210 let mut active_liquidity = 0_u128;
2211
2212 for position in &snapshot.positions {
2213 if position.tick_lower >= position.tick_upper
2214 || position.tick_lower < PoolTick::MIN_TICK
2215 || position.tick_upper > PoolTick::MAX_TICK
2216 || position.tick_lower % tick_spacing != 0
2217 || position.tick_upper % tick_spacing != 0
2218 {
2219 anyhow::bail!(
2220 "RPC position {} has invalid tick range [{}, {}) for spacing {tick_spacing}",
2221 position.owner,
2222 position.tick_lower,
2223 position.tick_upper
2224 );
2225 }
2226
2227 if position.liquidity == 0 {
2228 continue;
2229 }
2230
2231 let liquidity_net = i128::try_from(position.liquidity).with_context(|| {
2232 format!(
2233 "RPC position {} liquidity exceeds i128::MAX",
2234 position.owner
2235 )
2236 })?;
2237 let lower = derived_ticks.entry(position.tick_lower).or_default();
2238 lower.0 = lower
2239 .0
2240 .checked_add(position.liquidity)
2241 .context("lower tick liquidity gross overflow")?;
2242 lower.1 = lower
2243 .1
2244 .checked_add(liquidity_net)
2245 .context("lower tick liquidity net overflow")?;
2246 let upper = derived_ticks.entry(position.tick_upper).or_default();
2247 upper.0 = upper
2248 .0
2249 .checked_add(position.liquidity)
2250 .context("upper tick liquidity gross overflow")?;
2251 upper.1 = upper
2252 .1
2253 .checked_sub(liquidity_net)
2254 .context("upper tick liquidity net underflow")?;
2255
2256 if position.tick_lower <= snapshot.state.current_tick
2257 && snapshot.state.current_tick < position.tick_upper
2258 {
2259 active_liquidity = active_liquidity
2260 .checked_add(position.liquidity)
2261 .context("active position liquidity overflow")?;
2262 }
2263 }
2264
2265 if active_liquidity != snapshot.state.liquidity {
2266 anyhow::bail!(
2267 "RPC active liquidity mismatch: positions sum to {active_liquidity}, global state reports {}",
2268 snapshot.state.liquidity
2269 );
2270 }
2271
2272 if derived_ticks.len() != actual_ticks.len() {
2273 anyhow::bail!(
2274 "RPC tick topology does not match positions: derived {} ticks, received {}",
2275 derived_ticks.len(),
2276 actual_ticks.len()
2277 );
2278 }
2279
2280 for (tick_value, (liquidity_gross, liquidity_net)) in derived_ticks {
2281 let actual_tick = actual_ticks
2282 .get(&tick_value)
2283 .with_context(|| format!("RPC snapshot omitted position boundary {tick_value}"))?;
2284
2285 if actual_tick.liquidity_gross != liquidity_gross
2286 || actual_tick.liquidity_net != liquidity_net
2287 {
2288 anyhow::bail!(
2289 "RPC tick {tick_value} does not match positions: derived gross={liquidity_gross} net={liquidity_net}, received gross={} net={}",
2290 actual_tick.liquidity_gross,
2291 actual_tick.liquidity_net
2292 );
2293 }
2294 }
2295
2296 Ok(())
2297 }
2298
2299 fn timestamp_for_on_chain_snapshot(
2300 profiler: &PoolProfiler,
2301 cached_timestamp: Option<UnixNanos>,
2302 ) -> anyhow::Result<UnixNanos> {
2303 cached_timestamp
2304 .or(profiler.last_processed_ts)
2305 .context("missing block timestamp for on-chain snapshot")
2306 }
2307
2308 fn last_processed_event_for_on_chain_snapshot(
2309 profiler: &PoolProfiler,
2310 ) -> anyhow::Result<BlockPosition> {
2311 profiler.last_processed_event.clone().with_context(|| {
2312 format!(
2313 "cannot fetch on-chain snapshot for pool {} without a processed event",
2314 profiler.pool.address
2315 )
2316 })
2317 }
2318
2319 async fn block_scoped_snapshot_position(
2320 &mut self,
2321 block_number: u64,
2322 ) -> anyhow::Result<BlockPosition> {
2323 let blocks_stream = self
2324 .hypersync_client
2325 .request_blocks_stream(block_number, Some(block_number))
2326 .await;
2327 tokio::pin!(blocks_stream);
2328 let block = blocks_stream
2329 .next()
2330 .await
2331 .with_context(|| format!("failed to fetch block {block_number} for RPC snapshot"))?;
2332
2333 let block_position =
2334 Self::block_scoped_snapshot_position_from_block(&mut self.cache, &block, block_number)?;
2335 self.cache.add_pool_event_blocks_batch(vec![block]).await?;
2336
2337 Ok(block_position)
2338 }
2339
2340 fn block_scoped_snapshot_position_from_block(
2341 cache: &mut BlockchainCache,
2342 block: &Block,
2343 block_number: u64,
2344 ) -> anyhow::Result<BlockPosition> {
2345 if block.number != block_number {
2346 anyhow::bail!(
2347 "Fetched block {} while requesting RPC snapshot block {}",
2348 block.number,
2349 block_number
2350 );
2351 }
2352
2353 cache.cache_block_metadata(block)?;
2354
2355 Ok(BlockPosition::new(
2356 block.number,
2357 block.hash.clone(),
2358 BLOCK_SCOPED_SNAPSHOT_INDEX,
2359 BLOCK_SCOPED_SNAPSHOT_INDEX,
2360 )
2361 .with_block_hash(Some(block.hash.clone())))
2362 }
2363
2364 pub async fn replay_pool_events(&self, pool: &Pool, dex: &SharedDex) -> anyhow::Result<()> {
2373 if let Some(database) = &self.cache.database {
2374 log::debug!(
2375 "Replaying historical events for pool {} to hydrate profiler",
2376 pool.instrument_id
2377 );
2378
2379 let mut event_stream = database.stream_pool_events(
2380 Arc::clone(&self.chain),
2381 Arc::clone(dex),
2382 pool.instrument_id,
2383 pool.pool_identifier,
2384 None,
2385 None,
2386 );
2387 let mut event_count = 0;
2388
2389 while let Some(event_result) = event_stream.next().await {
2390 match event_result {
2391 Ok(event) => {
2392 let data_event = match event {
2393 DexPoolData::Swap(swap) => DataEvent::DeFi(DefiData::PoolSwap(swap)),
2394 DexPoolData::LiquidityUpdate(update) => {
2395 DataEvent::DeFi(DefiData::PoolLiquidityUpdate(update))
2396 }
2397 DexPoolData::FeeCollect(collect) => {
2398 DataEvent::DeFi(DefiData::PoolFeeCollect(collect))
2399 }
2400 DexPoolData::FeeProtocolUpdate(update) => {
2401 DataEvent::DeFi(DefiData::PoolFeeProtocolUpdate(update))
2402 }
2403 DexPoolData::FeeProtocolCollect(collect) => {
2404 DataEvent::DeFi(DefiData::PoolFeeProtocolCollect(collect))
2405 }
2406 DexPoolData::Flash(flash) => {
2407 DataEvent::DeFi(DefiData::PoolFlash(flash))
2408 }
2409 };
2410 self.send_data(data_event);
2411 event_count += 1;
2412 }
2413 Err(e) => {
2414 log::error!("Error streaming event for pool {}: {e}", pool.instrument_id);
2415 }
2416 }
2417 }
2418
2419 log::debug!(
2420 "Replayed {event_count} historical events for pool {}",
2421 pool.instrument_id
2422 );
2423 } else {
2424 log::debug!(
2425 "No database available, skipping event replay for pool {}",
2426 pool.instrument_id
2427 );
2428 }
2429
2430 Ok(())
2431 }
2432
2433 fn determine_from_block(&self) -> u64 {
2435 self.config
2436 .from_block
2437 .unwrap_or_else(|| self.cache.min_dex_creation_block().unwrap_or(0))
2438 }
2439
2440 fn get_dex_extended(&self, dex_id: &DexType) -> anyhow::Result<&DexExtended> {
2442 if !self.cache.get_registered_dexes().contains(dex_id) {
2443 anyhow::bail!("DEX {dex_id} is not registered in the data client");
2444 }
2445
2446 match get_dex_extended(self.chain.name, dex_id) {
2447 Some(dex) => Ok(dex),
2448 None => anyhow::bail!("Dex {dex_id} doesn't exist for chain {}", self.chain.name),
2449 }
2450 }
2451
2452 pub fn get_pool(&self, pool_identifier: &PoolIdentifier) -> anyhow::Result<&SharedPool> {
2458 match self.cache.get_pool(pool_identifier) {
2459 Some(pool) => Ok(pool),
2460 None => anyhow::bail!("Pool {pool_identifier} is not registered"),
2461 }
2462 }
2463
2464 pub fn send_data(&self, data: DataEvent) {
2466 if let Some(data_tx) = &self.data_tx {
2467 log::debug!("Sending {data}");
2468
2469 if let Err(e) = data_tx.send(data) {
2470 log::error!("Failed to send data: {e}");
2471 }
2472 } else {
2473 log::error!("No data event channel for sending data");
2474 }
2475 }
2476
2477 pub async fn disconnect(&mut self) {
2482 self.subscription_manager.clear_block_demand();
2483 self.hypersync_client.disconnect().await;
2484 }
2485}
2486
2487#[cfg(test)]
2488mod tests {
2489 use std::str::FromStr;
2490
2491 use alloy::primitives::{Address, B256, I256, U160, U256, address};
2492 use nautilus_core::UnixNanos;
2493 use nautilus_model::defi::{
2494 Chain, Token,
2495 pool_analysis::{
2496 position::PoolPosition,
2497 snapshot::{PoolAnalytics, PoolState},
2498 },
2499 };
2500 use rstest::rstest;
2501 use tokio_util::sync::CancellationToken;
2502 use ustr::Ustr;
2503
2504 use super::*;
2505
2506 const WETH_USDT_CREATION_BLOCK: u64 = 12_375_326;
2507
2508 #[rstest]
2509 #[case(SnapshotValidation::OnChain, "on_chain", true)]
2510 #[case(SnapshotValidation::Replay, "replay", true)]
2511 #[case(SnapshotValidation::Invalid, "invalid", false)]
2512 fn snapshot_validation_db_token_and_usability(
2513 #[case] validation: SnapshotValidation,
2514 #[case] expected_str: &str,
2515 #[case] expected_usable: bool,
2516 ) {
2517 assert_eq!(validation.as_str(), expected_str);
2520 assert_eq!(validation.is_usable(), expected_usable);
2521 assert_eq!(
2524 SnapshotValidation::from_db_token(expected_str),
2525 Some(validation)
2526 );
2527 }
2528
2529 #[rstest]
2530 fn snapshot_validation_from_db_token_rejects_unknown() {
2531 assert_eq!(SnapshotValidation::from_db_token("bogus"), None);
2532 }
2533
2534 #[rstest]
2535 fn last_processed_event_for_on_chain_snapshot_rejects_unprocessed_profiler() {
2536 let mut profiler = PoolProfiler::new(weth_usdt_pool());
2537 profiler
2538 .initialize(U160::from_str_radix("3cb0adde486484998be0b", 16).unwrap())
2539 .expect("Known WETH/USDT initial sqrt price should initialize");
2540
2541 let error = BlockchainDataClientCore::last_processed_event_for_on_chain_snapshot(&profiler)
2542 .expect_err("unprocessed profiler should not fetch on-chain state");
2543
2544 assert_eq!(
2545 error.to_string(),
2546 format!(
2547 "cannot fetch on-chain snapshot for pool {} without a processed event",
2548 profiler.pool.address
2549 )
2550 );
2551 }
2552
2553 #[rstest]
2554 fn validate_rpc_snapshot_topology_accepts_consistent_snapshot() {
2555 let (profiler, snapshot) = rpc_topology_fixture();
2556
2557 let result = BlockchainDataClientCore::validate_rpc_snapshot_topology(&profiler, &snapshot);
2558
2559 assert!(result.is_ok());
2560 }
2561
2562 #[rstest]
2563 fn validate_rpc_snapshot_topology_rejects_missing_tick() {
2564 let (profiler, mut snapshot) = rpc_topology_fixture();
2565 snapshot.ticks.pop();
2566
2567 let error = BlockchainDataClientCore::validate_rpc_snapshot_topology(&profiler, &snapshot)
2568 .unwrap_err();
2569
2570 assert!(error.to_string().contains("expected 2 ticks, received 1"));
2571 }
2572
2573 #[rstest]
2574 fn validate_rpc_snapshot_topology_rejects_zeroed_position() {
2575 let (profiler, mut snapshot) = rpc_topology_fixture();
2576 snapshot.positions[0].liquidity = 0;
2577
2578 let error = BlockchainDataClientCore::validate_rpc_snapshot_topology(&profiler, &snapshot)
2579 .unwrap_err();
2580
2581 assert!(
2582 error
2583 .to_string()
2584 .contains("RPC positions do not match the complete HyperSync topology")
2585 );
2586 }
2587
2588 #[rstest]
2589 fn validate_rpc_snapshot_topology_rejects_active_liquidity_mismatch() {
2590 let (profiler, mut snapshot) = rpc_topology_fixture();
2591 snapshot.state.liquidity -= 1;
2592
2593 let error = BlockchainDataClientCore::validate_rpc_snapshot_topology(&profiler, &snapshot)
2594 .unwrap_err();
2595
2596 assert!(error.to_string().contains("RPC active liquidity mismatch"));
2597 }
2598
2599 #[rstest]
2600 fn validate_rpc_snapshot_topology_rejects_tick_liquidity_mismatch() {
2601 let (profiler, mut snapshot) = rpc_topology_fixture();
2602 snapshot.ticks[0].liquidity_gross -= 1;
2603
2604 let error = BlockchainDataClientCore::validate_rpc_snapshot_topology(&profiler, &snapshot)
2605 .unwrap_err();
2606
2607 assert!(error.to_string().contains("RPC tick -60 topology mismatch"));
2608 }
2609
2610 #[rstest]
2611 fn timestamp_for_on_chain_snapshot_prefers_cached_block_timestamp() {
2612 let pool = weth_usdt_pool();
2613 let mut profiler = PoolProfiler::new(pool);
2614 let cached_ts = UnixNanos::from(1_700_000_001_000_000_000);
2615 let profiler_ts = UnixNanos::from(1_700_000_000_000_000_000);
2616 profiler.last_processed_ts = Some(profiler_ts);
2617
2618 let timestamp =
2619 BlockchainDataClientCore::timestamp_for_on_chain_snapshot(&profiler, Some(cached_ts))
2620 .unwrap();
2621
2622 assert_eq!(timestamp, cached_ts);
2623 }
2624
2625 #[rstest]
2626 fn timestamp_for_on_chain_snapshot_falls_back_to_profiler_timestamp() {
2627 let pool = weth_usdt_pool();
2628 let mut profiler = PoolProfiler::new(pool);
2629 let profiler_ts = UnixNanos::from(1_700_000_000_000_000_000);
2630 profiler.last_processed_ts = Some(profiler_ts);
2631
2632 let timestamp =
2633 BlockchainDataClientCore::timestamp_for_on_chain_snapshot(&profiler, None).unwrap();
2634
2635 assert_eq!(timestamp, profiler_ts);
2636 }
2637
2638 #[rstest]
2639 fn timestamp_for_on_chain_snapshot_rejects_missing_timestamp() {
2640 let pool = weth_usdt_pool();
2641 let profiler = PoolProfiler::new(pool);
2642
2643 let error = BlockchainDataClientCore::timestamp_for_on_chain_snapshot(&profiler, None)
2644 .expect_err("missing timestamps should fail");
2645
2646 assert_eq!(
2647 error.to_string(),
2648 "missing block timestamp for on-chain snapshot"
2649 );
2650 }
2651
2652 #[rstest]
2653 fn block_scoped_snapshot_position_from_block_caches_timestamp_and_uses_sentinel_indexes() {
2654 let chain = Arc::new(
2655 Chain::from_chain_id(42161)
2656 .expect("Arbitrum chain should exist")
2657 .clone(),
2658 );
2659 let mut cache = BlockchainCache::new(chain);
2660 let timestamp = UnixNanos::from(1_700_000_002_000_000_000);
2661 let block = test_block(123, timestamp);
2662
2663 let position = BlockchainDataClientCore::block_scoped_snapshot_position_from_block(
2664 &mut cache, &block, 123,
2665 )
2666 .unwrap();
2667
2668 assert_eq!(position.number, 123);
2669 assert_eq!(position.transaction_hash, block.hash);
2670 assert_eq!(position.transaction_index, BLOCK_SCOPED_SNAPSHOT_INDEX);
2671 assert_eq!(position.log_index, BLOCK_SCOPED_SNAPSHOT_INDEX);
2672 assert_eq!(cache.get_block_timestamp(123), Some(×tamp));
2673 }
2674
2675 #[rstest]
2676 fn block_scoped_snapshot_position_from_block_rejects_mismatched_block() {
2677 let chain = Arc::new(
2678 Chain::from_chain_id(42161)
2679 .expect("Arbitrum chain should exist")
2680 .clone(),
2681 );
2682 let mut cache = BlockchainCache::new(chain);
2683 let block = test_block(122, UnixNanos::from(1_700_000_002_000_000_000));
2684
2685 let error = BlockchainDataClientCore::block_scoped_snapshot_position_from_block(
2686 &mut cache, &block, 123,
2687 )
2688 .expect_err("mismatched block should fail");
2689
2690 assert_eq!(
2691 error.to_string(),
2692 "Fetched block 122 while requesting RPC snapshot block 123"
2693 );
2694 assert_eq!(cache.get_block_timestamp(122), None);
2695 }
2696
2697 #[rstest]
2698 #[case(100, 50, Some(99))]
2699 #[case(50, 50, None)]
2700 #[case(0, 0, None)]
2701 fn completed_pool_event_checkpoint_excludes_in_flight_block(
2702 #[case] block_number: u64,
2703 #[case] effective_from_block: u64,
2704 #[case] expected: Option<u64>,
2705 ) {
2706 let checkpoint = BlockchainDataClientCore::completed_pool_event_checkpoint(
2707 block_number,
2708 effective_from_block,
2709 );
2710
2711 assert_eq!(checkpoint, expected);
2712 }
2713
2714 #[rstest]
2715 fn pool_event_sync_ranges_preserve_legacy_history_during_new_family_backfill() {
2716 let families = vec![
2717 PoolEventFamily {
2718 name: "swap",
2719 signature: "swap".to_string(),
2720 introduced_version: 0,
2721 },
2722 PoolEventFamily {
2723 name: "fee_protocol_update",
2724 signature: "fee_protocol_update".to_string(),
2725 introduced_version: 1,
2726 },
2727 PoolEventFamily {
2728 name: "fee_protocol_collect",
2729 signature: "fee_protocol_collect".to_string(),
2730 introduced_version: 1,
2731 },
2732 ];
2733 let state = PoolEventSyncState {
2734 version: 0,
2735 last_full_sync_block: Some(100),
2736 family_blocks: Vec::new(),
2737 };
2738
2739 let ranges = BlockchainDataClientCore::pool_event_sync_ranges(&families, &state, 10, 110);
2740
2741 assert_eq!(
2742 ranges,
2743 vec![
2744 PoolEventSyncRange {
2745 from_block: 10,
2746 families: vec![families[1].clone(), families[2].clone()],
2747 },
2748 PoolEventSyncRange {
2749 from_block: 101,
2750 families: vec![families[0].clone()],
2751 },
2752 ]
2753 );
2754 }
2755
2756 #[rstest]
2757 fn pool_event_sync_ranges_group_families_by_checkpoint() {
2758 let families = vec![
2759 PoolEventFamily {
2760 name: "swap",
2761 signature: "swap".to_string(),
2762 introduced_version: 0,
2763 },
2764 PoolEventFamily {
2765 name: "fee_protocol_update",
2766 signature: "fee_protocol_update".to_string(),
2767 introduced_version: 1,
2768 },
2769 PoolEventFamily {
2770 name: "fee_protocol_collect",
2771 signature: "fee_protocol_collect".to_string(),
2772 introduced_version: 1,
2773 },
2774 ];
2775 let state = PoolEventSyncState {
2776 version: 1,
2777 last_full_sync_block: Some(100),
2778 family_blocks: vec![
2779 ("fee_protocol_collect".to_string(), 75),
2780 ("fee_protocol_update".to_string(), 50),
2781 ("swap".to_string(), 100),
2782 ],
2783 };
2784
2785 let ranges = BlockchainDataClientCore::pool_event_sync_ranges(&families, &state, 10, 100);
2786
2787 assert_eq!(
2788 ranges,
2789 vec![
2790 PoolEventSyncRange {
2791 from_block: 51,
2792 families: vec![families[1].clone()],
2793 },
2794 PoolEventSyncRange {
2795 from_block: 76,
2796 families: vec![families[2].clone()],
2797 },
2798 ]
2799 );
2800 }
2801
2802 #[rstest]
2803 fn pool_event_sync_ranges_backfill_future_family_without_schema_migration() {
2804 let families = vec![
2805 PoolEventFamily {
2806 name: "swap",
2807 signature: "swap".to_string(),
2808 introduced_version: 0,
2809 },
2810 PoolEventFamily {
2811 name: "future",
2812 signature: "future".to_string(),
2813 introduced_version: 2,
2814 },
2815 ];
2816 let state = PoolEventSyncState {
2817 version: 1,
2818 last_full_sync_block: Some(100),
2819 family_blocks: vec![("swap".to_string(), 100)],
2820 };
2821
2822 let ranges = BlockchainDataClientCore::pool_event_sync_ranges(&families, &state, 10, 100);
2823
2824 assert_eq!(
2825 ranges,
2826 vec![PoolEventSyncRange {
2827 from_block: 10,
2828 families: vec![families[1].clone()],
2829 }]
2830 );
2831 }
2832
2833 #[rstest]
2834 fn process_pool_swap_event_rejects_mismatched_block_hash() {
2835 let Some(mut core) = test_core().expect("test core builds") else {
2836 return;
2837 };
2838
2839 let pool = weth_usdt_pool();
2840 let block = test_block_with_hash(
2841 42,
2842 "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
2843 UnixNanos::from(1_700_000_000_000_000_000),
2844 );
2845 core.cache
2846 .cache_block_metadata(&block)
2847 .expect("test block hash must be valid");
2848
2849 let event = test_swap_event(
2850 &pool,
2851 42,
2852 Some("0xBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"),
2853 );
2854 let error = core
2855 .process_pool_swap_event(&event, &pool)
2856 .expect_err("mismatched observed hash should not convert");
2857
2858 assert_eq!(
2859 error.to_string(),
2860 "block hash mismatch for swap event at block 42: observed \
2861 0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb, cached \
2862 0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
2863 );
2864 }
2865
2866 #[rstest]
2867 fn process_pool_swap_event_attaches_matching_block_hash_and_timestamp() {
2868 let Some(mut core) = test_core().expect("test core builds") else {
2869 return;
2870 };
2871
2872 let pool = weth_usdt_pool();
2873 let timestamp = UnixNanos::from(1_700_000_000_000_000_001);
2874 let block = test_block_with_hash(
2875 42,
2876 "0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
2877 timestamp,
2878 );
2879 core.cache
2880 .cache_block_metadata(&block)
2881 .expect("test block hash must be valid");
2882
2883 let event = test_swap_event(
2884 &pool,
2885 42,
2886 Some("0xBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"),
2887 );
2888 let swap = core
2889 .process_pool_swap_event(&event, &pool)
2890 .expect("matching observed hash converts");
2891
2892 assert_eq!(swap.block, 42);
2893 assert_eq!(
2894 swap.block_hash,
2895 Some("0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb".to_string())
2896 );
2897 assert_eq!(swap.ts_event, timestamp);
2898 assert_eq!(swap.ts_init, timestamp);
2899 }
2900
2901 fn weth_usdt_pool() -> SharedPool {
2902 let chain = Arc::new(
2903 Chain::from_chain_id(1)
2904 .expect("Ethereum chain should exist")
2905 .clone(),
2906 );
2907 let dex = Arc::clone(
2908 &get_dex_extended(chain.name, &DexType::UniswapV3)
2909 .expect("Ethereum UniswapV3 should be registered")
2910 .dex,
2911 );
2912 let pool_address = address!("4e68ccd3e89f51c3074ca5072bbac773960dfa36");
2913 let token0 = Token::new(
2914 Arc::clone(&chain),
2915 address!("C02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2"),
2916 "Wrapped Ether".to_string(),
2917 "WETH".to_string(),
2918 18,
2919 );
2920 let token1 = Token::new(
2921 Arc::clone(&chain),
2922 address!("dAC17F958D2ee523a2206206994597C13D831ec7"),
2923 "Tether USD".to_string(),
2924 "USDT".to_string(),
2925 6,
2926 );
2927
2928 Arc::new(Pool::new(
2929 chain,
2930 dex,
2931 pool_address,
2932 PoolIdentifier::from_address(pool_address),
2933 WETH_USDT_CREATION_BLOCK,
2934 token0,
2935 token1,
2936 Some(3_000),
2937 Some(60),
2938 UnixNanos::default(),
2939 ))
2940 }
2941
2942 fn rpc_topology_fixture() -> (PoolProfiler, PoolSnapshot) {
2943 let pool = weth_usdt_pool();
2944 let owner = Address::ZERO;
2945 let liquidity = 1_000_u128;
2946 let state = PoolState {
2947 current_tick: 0,
2948 liquidity,
2949 ..Default::default()
2950 };
2951 let position = PoolPosition::new(owner, -60, 60, liquidity as i128);
2952 let ticks = vec![
2953 PoolTick::new(
2954 -60,
2955 liquidity,
2956 liquidity as i128,
2957 U256::ZERO,
2958 U256::ZERO,
2959 true,
2960 0,
2961 ),
2962 PoolTick::new(
2963 60,
2964 liquidity,
2965 -(liquidity as i128),
2966 U256::ZERO,
2967 U256::ZERO,
2968 true,
2969 0,
2970 ),
2971 ];
2972 let timestamp = UnixNanos::from(1_700_000_000_000_000_000);
2973 let snapshot = PoolSnapshot::new(
2974 pool.instrument_id,
2975 state,
2976 vec![position],
2977 ticks,
2978 PoolAnalytics::default(),
2979 BlockPosition::new(100, "0xabc".to_string(), 0, 0),
2980 timestamp,
2981 timestamp,
2982 );
2983 let mut profiler = PoolProfiler::new(pool);
2984 profiler.restore_from_snapshot(snapshot.clone()).unwrap();
2985
2986 (profiler, snapshot)
2987 }
2988
2989 fn test_block(number: u64, timestamp: UnixNanos) -> Block {
2990 Block::new(
2991 format!("0x{number:064x}"),
2992 String::from("0x0"),
2993 number,
2994 Ustr::from("0x0000000000000000000000000000000000000000"),
2995 30_000_000,
2996 21_000,
2997 timestamp,
2998 Some(Blockchain::Arbitrum),
2999 )
3000 }
3001
3002 fn test_block_with_hash(number: u64, hash: &str, timestamp: UnixNanos) -> Block {
3003 Block::new(
3004 hash.to_string(),
3005 String::from("0x0"),
3006 number,
3007 Ustr::from("0x0000000000000000000000000000000000000000"),
3008 30_000_000,
3009 21_000,
3010 timestamp,
3011 Some(Blockchain::Ethereum),
3012 )
3013 }
3014
3015 fn test_core() -> anyhow::Result<Option<BlockchainDataClientCore>> {
3016 if std::env::var("ENVIO_API_TOKEN").is_err() {
3017 return Ok(None);
3018 }
3019
3020 let chain = Arc::new(
3021 Chain::from_chain_id(1)
3022 .expect("Ethereum chain should exist")
3023 .clone(),
3024 );
3025 let config = BlockchainDataClientConfig::builder()
3026 .chain(Arc::clone(&chain))
3027 .dex_ids(vec![DexType::UniswapV3])
3028 .http_rpc_url("http://127.0.0.1:9".into())
3029 .build();
3030 let core = BlockchainDataClientCore::new(config, None, None, CancellationToken::new())?;
3031
3032 Ok(Some(core))
3033 }
3034
3035 fn fixture_hash(hash: &str) -> B256 {
3036 B256::from_str(hash).expect("fixture block hash must be valid")
3037 }
3038
3039 fn test_swap_event(
3040 pool: &SharedPool,
3041 block_number: u64,
3042 block_hash: Option<&str>,
3043 ) -> SwapEvent {
3044 let address = address!("1111111111111111111111111111111111111111");
3045
3046 SwapEvent::new(
3047 Arc::clone(&pool.dex),
3048 pool.pool_identifier,
3049 block_number,
3050 block_hash.map(fixture_hash),
3051 "0xdeadbeef".to_string(),
3052 0,
3053 7,
3054 address,
3055 address,
3056 I256::ZERO,
3057 I256::ZERO,
3058 U160::ZERO,
3059 0,
3060 0,
3061 )
3062 }
3063}