diff --git a/Cargo.toml b/Cargo.toml index 9b46e02..095b949 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ alloy = { version = "1.4", features = ["full", "rpc-types-trace"] } indicatif = "0.18" reqwest = { version = "0.12", features = ["json", "gzip"] } axum = "0.8.9" -tower-http = { version = "0.6.10", features = ["fs", "cors", "trace"] } +tower-http = { version = "0.6.10", features = ["fs", "cors", "trace", "compression-gzip"] } duckdb = { version = "1.10", features = ["bundled"] } tracing = "0.1.44" tracing-subscriber = { version = "0.3.23", features = ["env-filter", "chrono"] } diff --git a/src/blocks.rs b/src/blocks.rs index 5ebb5fb..b565039 100644 --- a/src/blocks.rs +++ b/src/blocks.rs @@ -156,35 +156,3 @@ pub fn blocks_per_day(chain_id: u64, block_number: u64) -> u64 { } } } - -#[cfg(test)] -mod tests { - use chrono::{Datelike, TimeZone}; - - use super::{block_number_at_time, block_timestamp}; - use crate::chains::GNOSIS_CHAIN_ID; - - #[test] - fn gnosis_recent_blocks_map_to_current_calendar_dates() { - assert_eq!( - block_timestamp(GNOSIS_CHAIN_ID, 46_762_380), - chrono::Utc - .with_ymd_and_hms(2026, 6, 18, 16, 0, 38) - .unwrap() - ); - } - - #[test] - fn gnosis_recent_time_ranges_convert_back_to_blocks() { - let block = block_number_at_time( - GNOSIS_CHAIN_ID, - chrono::Utc.with_ymd_and_hms(2026, 6, 19, 0, 0, 0).unwrap(), - ); - - assert!(block > 46_762_380); - assert_eq!( - block_timestamp(GNOSIS_CHAIN_ID, block).date_naive().month(), - 6 - ); - } -} diff --git a/src/cli.rs b/src/cli.rs index a152102..1135449 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -133,6 +133,9 @@ pub struct LoadArgs { /// Rebuild existing CSV import tables or replace existing Parquet links #[arg(long)] pub overwrite: bool, + /// Rebuild all Verifier Alliance labels instead of importing changed VA files only + #[arg(long = "rebuild-va", alias = "rebuild")] + pub rebuild_va: bool, /// DuckDB memory limit for CSV and verification imports #[arg(long, default_value = "8GB")] pub memory_limit: String, @@ -190,4 +193,14 @@ pub struct ServeArgs { /// Background extraction max concurrent HTTP requests #[arg(long, default_value_t = 16)] pub tail_max_concurrent_requests: usize, + /// DuckDB memory limit for the dashboard database, e.g. "2GB". + /// Recommended on small hosts; DuckDB otherwise assumes 80% of RAM. + #[arg(long, env = "BLINK_DB_MEMORY_LIMIT")] + pub db_memory_limit: Option, + /// DuckDB thread count for the dashboard database + #[arg(long, env = "BLINK_DB_THREADS")] + pub db_threads: Option, + /// Read connections serving dashboard queries (0 = auto-size from cores) + #[arg(long, env = "BLINK_DB_READERS", default_value_t = 0)] + pub db_readers: usize, } diff --git a/src/db.rs b/src/db.rs deleted file mode 100644 index ff197a4..0000000 --- a/src/db.rs +++ /dev/null @@ -1,3225 +0,0 @@ -//! DuckDB-backed query layer for the dashboard. -//! -//! Owns a single persistent DuckDB connection (file: `{data_dir}/blink.duckdb`) -//! that exposes: -//! - a `contracts` view over every `*.parquet` file in the data directory -//! (multi-source: blink, cryo, paradigm — `union_by_name = true`); -//! - an `enrichment` table populated by bulk verification-registry imports. -//! -//! All query methods return owned, JSON-serializable structs and run on -//! `spawn_blocking` so axum handlers stay async. The connection is wrapped -//! in a `tokio::sync::Mutex`; queries serialize on it, which is acceptable -//! for an analytics dashboard's request rate. - -use std::{ - collections::HashSet, - path::{Path, PathBuf}, - sync::Arc, - time::Instant, -}; - -use anyhow::{anyhow, Context, Result}; -use chrono::{DateTime, Utc}; -use duckdb::{params, types::ValueRef, AccessMode, Config, Connection, Row}; -use serde_json::{Number, Value}; -use tokio::sync::Mutex; - -use crate::{chains::ETHEREUM_CHAIN_ID, util::match_simple_glob}; - -const RECENT_PARQUET_FILE_LIMIT: usize = 12; - -#[derive(Clone)] -pub struct Db { - inner: Arc>, - data_dir: PathBuf, - contracts_glob: String, -} - -#[derive(Debug, Clone, serde::Serialize, utoipa::ToSchema)] -pub struct Stats { - pub total_contracts: u64, - pub verified_count: u64, - pub unverified_count: u64, - pub verified_pct: f64, - pub last_block: u64, - pub first_block: u64, - pub enrichment_coverage_pct: f64, - pub last_updated: DateTime, -} - -#[derive(Debug, Clone, serde::Serialize, utoipa::ToSchema)] -pub struct DeployBucket { - pub block_start: u64, - pub block_end: u64, - pub timestamp: DateTime, - pub count: u64, -} - -#[derive(Debug, Clone, serde::Serialize, utoipa::ToSchema)] -pub struct VerifiedRatioBucket { - pub block_start: u64, - pub block_end: u64, - pub timestamp: DateTime, - pub verified: u64, - pub unverified: u64, - pub unknown: u64, -} - -#[derive(Debug, Clone, serde::Serialize, utoipa::ToSchema)] -pub struct SizeBin { - pub label: String, - pub size_min: u64, - pub size_max: u64, - pub count: u64, -} - -#[derive(Debug, Clone, serde::Serialize, utoipa::ToSchema)] -pub struct CompilerCount { - pub compiler_version: String, - pub count: u64, -} - -#[derive(Debug, Clone, serde::Serialize, utoipa::ToSchema)] -pub struct LanguageCount { - pub language: String, - pub count: u64, -} - -#[derive(Debug, Clone, serde::Serialize, utoipa::ToSchema)] -pub struct StandardsBreakdown { - pub erc20: u64, - pub erc721: u64, - pub erc1155: u64, - pub proxy_eip1967: u64, - pub proxy_minimal: u64, - pub uses_push0: u64, - pub has_source_hash: u64, - pub total_decoded: u64, -} - -#[derive(Debug, Clone, serde::Serialize, utoipa::ToSchema)] -pub struct RecentContract { - pub address: String, - pub block_number: u64, - pub create_index: u64, - pub timestamp: DateTime, - pub deployer: String, - pub n_code_bytes: u64, - pub is_verified: Option, - pub contract_name: Option, - pub compiler_version: Option, -} - -#[derive(Debug, Clone, Copy)] -pub struct RecentCursor { - pub block_number: u64, - pub create_index: u64, -} - -#[derive(Debug, Clone, serde::Serialize, utoipa::ToSchema)] -pub struct RecentPage { - pub contracts: Vec, - pub has_more: bool, -} - -#[derive(Debug, Clone, serde::Serialize, utoipa::ToSchema)] -pub struct SqlQueryResult { - pub columns: Vec, - #[schema(value_type = Vec>)] - pub rows: Vec>, - pub row_count: usize, - pub limit: u32, - pub elapsed_ms: u128, -} - -type RecentRowData = ( - Vec, - u32, - u32, - Option>, - Option, - Option, - Option, - Option, -); - -fn read_recent_row(row: &Row<'_>) -> duckdb::Result { - Ok(( - row.get::<_, Vec>(0)?, - row.get::<_, u32>(1)?, - row.get::<_, u32>(2)?, - row.get::<_, Option>>(3)?, - row.get::<_, Option>(4)?, - row.get::<_, Option>(5)?, - row.get::<_, Option>(6)?, - row.get::<_, Option>(7)?, - )) -} - -fn read_u64_pair(row: &Row<'_>) -> duckdb::Result<(u64, u64)> { - Ok((row.get::<_, u64>(0)?, row.get::<_, u64>(1)?)) -} - -fn read_verified_bucket_row(row: &Row<'_>) -> duckdb::Result<(u64, u64, u64, u64)> { - Ok(( - row.get::<_, u64>(0)?, - row.get::<_, u64>(1)?, - row.get::<_, u64>(2)?, - row.get::<_, u64>(3)?, - )) -} - -fn read_string_u64_pair(row: &Row<'_>) -> duckdb::Result<(String, u64)> { - Ok((row.get::<_, String>(0)?, row.get::<_, u64>(1)?)) -} - -fn table_exists(conn: &Connection, table: &str) -> Result { - let count: i64 = conn - .query_row( - "SELECT COUNT(*) FROM information_schema.tables WHERE table_name = ?", - params![table], - |row| row.get(0), - ) - .unwrap_or(0); - Ok(count > 0) -} - -#[derive(Debug)] -struct ContractParquetFile { - path: PathBuf, - chain_id: Option, - start_block: Option, - end_block: Option, -} - -fn list_contract_parquet_files(data_dir: &Path, contracts_glob: &str) -> Result> { - let mut files: Vec = std::fs::read_dir(data_dir) - .with_context(|| format!("read data dir {}", data_dir.display()))? - .filter_map(|e| e.ok()) - .map(|e| e.path()) - .filter(|p| { - p.extension().and_then(|s| s.to_str()) == Some("parquet") - && match_simple_glob( - contracts_glob, - p.file_name().and_then(|s| s.to_str()).unwrap_or_default(), - ) - && p.file_name() - .and_then(|s| s.to_str()) - .map(|n| n != "enrichment.parquet") - .unwrap_or(true) - }) - .collect(); - files.sort(); - Ok(files) -} - -fn contract_file_with_range(path: PathBuf) -> ContractParquetFile { - let name = path - .file_name() - .and_then(|s| s.to_str()) - .unwrap_or_default(); - let nums = decimal_runs(name); - let chain_id = if name.starts_with("contracts__chain_") || name.starts_with("tail__chain_") { - nums.first().copied() - } else { - None - }; - let len = nums.len(); - let (start_block, end_block) = if len >= 2 { - (Some(nums[len - 2]), Some(nums[len - 1])) - } else { - (None, None) - }; - ContractParquetFile { - path, - chain_id, - start_block, - end_block, - } -} - -fn decimal_runs(input: &str) -> Vec { - let mut out = Vec::new(); - let mut current: Option = None; - for byte in input.bytes() { - if byte.is_ascii_digit() { - let digit = u64::from(byte - b'0'); - current = Some( - current - .unwrap_or(0) - .saturating_mul(10) - .saturating_add(digit), - ); - } else if let Some(value) = current.take() { - out.push(value); - } - } - if let Some(value) = current { - out.push(value); - } - out -} - -fn max_contract_file_block_for_chain(files: &[PathBuf], chain_id: u64) -> Option { - files - .iter() - .cloned() - .map(contract_file_with_range) - .filter(|file| { - file.chain_id == Some(chain_id) - || (file.chain_id.is_none() && chain_id == ETHEREUM_CHAIN_ID) - }) - .filter_map(|file| file.end_block) - .max() -} - -fn recent_contract_parquet_files( - files: &[PathBuf], - chain_id: u64, - cursor: Option, -) -> Vec { - let cursor_block = cursor.map(|cursor| cursor.block_number); - let mut ranged = files - .iter() - .cloned() - .map(contract_file_with_range) - .filter(|file| { - file.chain_id == Some(chain_id) - || (file.chain_id.is_none() && chain_id == ETHEREUM_CHAIN_ID) - }) - .collect::>(); - ranged.sort_by(|a, b| { - b.end_block - .unwrap_or(0) - .cmp(&a.end_block.unwrap_or(0)) - .then_with(|| b.start_block.unwrap_or(0).cmp(&a.start_block.unwrap_or(0))) - .then_with(|| b.path.cmp(&a.path)) - }); - - ranged - .into_iter() - .filter(|file| { - cursor_block - .map(|block| file.start_block.unwrap_or(0) <= block) - .unwrap_or(true) - }) - .take(RECENT_PARQUET_FILE_LIMIT) - .map(|file| file.path) - .collect() -} - -fn contract_parquet_files_for_block_range( - files: &[PathBuf], - chain_id: u64, - block_range: Option<(u64, u64)>, -) -> Vec { - let mut ranged = files - .iter() - .cloned() - .map(contract_file_with_range) - .filter(|file| { - file.chain_id == Some(chain_id) - || (file.chain_id.is_none() && chain_id == ETHEREUM_CHAIN_ID) - }) - .filter(|file| { - if let Some((start, end)) = block_range { - match (file.start_block, file.end_block) { - (Some(file_start), Some(file_end)) => file_end >= start && file_start <= end, - _ => true, - } - } else { - true - } - }) - .collect::>(); - ranged.sort_by(|a, b| { - a.start_block - .unwrap_or(0) - .cmp(&b.start_block.unwrap_or(0)) - .then_with(|| a.end_block.unwrap_or(0).cmp(&b.end_block.unwrap_or(0))) - .then_with(|| a.path.cmp(&b.path)) - }); - ranged.into_iter().map(|file| file.path).collect() -} - -fn parquet_read_list(files: &[PathBuf]) -> String { - files - .iter() - .map(|p| format!("'{}'", p.display().to_string().replace('\'', "''"))) - .collect::>() - .join(", ") -} - -fn create_contract_parquet_view( - conn: &Connection, - view_name: &str, - files: &[PathBuf], - chain_id: u64, -) -> Result<()> { - let body = if files.is_empty() { - r#" - SELECT - CAST(NULL AS UINTEGER) AS block_number, - CAST(NULL AS BLOB) AS block_hash, - CAST(NULL AS UINTEGER) AS create_index, - CAST(NULL AS BLOB) AS transaction_hash, - CAST(NULL AS BLOB) AS contract_address, - CAST(NULL AS BLOB) AS deployer, - CAST(NULL AS BLOB) AS factory, - CAST(NULL AS BLOB) AS init_code, - CAST(NULL AS BLOB) AS code, - CAST(NULL AS BLOB) AS init_code_hash, - CAST(NULL AS UINTEGER) AS n_init_code_bytes, - CAST(NULL AS UINTEGER) AS n_code_bytes, - CAST(NULL AS BLOB) AS code_hash, - CAST(NULL AS UBIGINT) AS chain_id - WHERE FALSE - "# - .to_string() - } else { - format!( - r#" - SELECT - block_number, block_hash, create_index, transaction_hash, - contract_address, deployer, factory, init_code, code, - init_code_hash, n_init_code_bytes, n_code_bytes, - code_hash, chain_id - FROM read_parquet([{}], union_by_name = true) - WHERE chain_id = {} - "#, - parquet_read_list(files), - chain_id - ) - }; - conn.execute_batch(&format!( - "CREATE OR REPLACE TEMP VIEW {view_name} AS\n{body};" - )) - .with_context(|| { - format!( - "create {view_name} parquet contracts view ({} files)", - files.len(), - ) - })?; - Ok(()) -} - -fn create_recent_parquet_contracts_view( - conn: &Connection, - files: &[PathBuf], - chain_id: u64, -) -> Result<()> { - create_contract_parquet_view(conn, "recent_parquet_contracts", files, chain_id) -} - -fn ensure_parquet_block_counts(conn: &Connection, files: &[PathBuf]) -> Result<()> { - conn.execute_batch( - r#" - CREATE TABLE IF NOT EXISTS parquet_block_counts ( - source_path VARCHAR NOT NULL, - chain_id UBIGINT NOT NULL, - block_number UINTEGER NOT NULL, - contract_count UBIGINT NOT NULL - ); - CREATE INDEX IF NOT EXISTS parquet_block_counts_chain_block_idx - ON parquet_block_counts(chain_id, block_number); - CREATE INDEX IF NOT EXISTS parquet_block_counts_source_idx - ON parquet_block_counts(source_path); - "#, - ) - .context("create parquet block counts table")?; - - let existing = conn - .prepare("SELECT DISTINCT source_path FROM parquet_block_counts")? - .query_map([], |row| row.get::<_, String>(0))? - .collect::, _>>()?; - let current = files - .iter() - .map(|path| path.display().to_string()) - .collect::>(); - for source_path in existing { - if !current.contains(&source_path) { - conn.execute( - "DELETE FROM parquet_block_counts WHERE source_path = ?", - params![source_path], - ) - .context("delete stale parquet block counts")?; - } - } - - let counted = conn - .prepare("SELECT DISTINCT source_path FROM parquet_block_counts")? - .query_map([], |row| row.get::<_, String>(0))? - .collect::, _>>()?; - for file in files { - let source_path = file.display().to_string(); - if counted.contains(&source_path) { - continue; - } - let source_path_sql = source_path.replace('\'', "''"); - conn.execute_batch(&format!( - r#" - INSERT INTO parquet_block_counts - SELECT - '{source_path_sql}' AS source_path, - chain_id::UBIGINT AS chain_id, - block_number::UINTEGER AS block_number, - COUNT(*)::UBIGINT AS contract_count - FROM read_parquet('{source_path_sql}', union_by_name = true) - WHERE block_number IS NOT NULL - AND chain_id IS NOT NULL - GROUP BY chain_id, block_number; - "# - )) - .with_context(|| format!("count parquet blocks in {}", file.display()))?; - } - - Ok(()) -} - -fn column_exists(conn: &Connection, table: &str, column: &str) -> Result { - let count: i64 = conn - .query_row( - r#" - SELECT COUNT(*) - FROM information_schema.columns - WHERE table_name = ? AND column_name = ? - "#, - params![table, column], - |row| row.get(0), - ) - .unwrap_or(0); - Ok(count > 0) -} - -fn contains_sql_keyword(sql: &str, keyword: &str) -> bool { - let bytes = sql.as_bytes(); - let needle = keyword.as_bytes(); - if needle.is_empty() || needle.len() > bytes.len() { - return false; - } - - bytes - .windows(needle.len()) - .enumerate() - .any(|(idx, window)| { - if window != needle { - return false; - } - let before = idx.checked_sub(1).and_then(|i| bytes.get(i)).copied(); - let after = bytes.get(idx + needle.len()).copied(); - !is_sql_ident_byte(before) && !is_sql_ident_byte(after) - }) -} - -fn is_sql_ident_byte(byte: Option) -> bool { - matches!(byte, Some(b'a'..=b'z' | b'0'..=b'9' | b'_')) -} - -fn normalize_read_only_sql(sql: &str) -> Result { - let trimmed = sql.trim(); - if trimmed.is_empty() { - return Err(anyhow!("query is empty")); - } - if trimmed.len() > 20_000 { - return Err(anyhow!("query is too large")); - } - - let without_trailing_semicolon = trimmed - .strip_suffix(';') - .map(str::trim_end) - .unwrap_or(trimmed); - if without_trailing_semicolon.contains(';') { - return Err(anyhow!("only one read-only statement is allowed")); - } - - let lower = without_trailing_semicolon.to_ascii_lowercase(); - let first = lower.split_whitespace().next().unwrap_or_default(); - if first != "select" && first != "with" { - return Err(anyhow!("only SELECT and WITH queries are allowed")); - } - - for keyword in [ - "alter", - "attach", - "call", - "checkpoint", - "copy", - "create", - "delete", - "detach", - "drop", - "export", - "import", - "insert", - "install", - "load", - "pragma", - "set", - "update", - "vacuum", - ] { - if contains_sql_keyword(&lower, keyword) { - return Err(anyhow!( - "keyword `{}` is not allowed in dashboard queries", - keyword - )); - } - } - - for function in [ - "read_blob", - "read_csv", - "read_json", - "read_parquet", - "csv_scan", - "parquet_scan", - "sqlite_scan", - "postgres_scan", - "mysql_scan", - "httpfs", - ] { - if lower.contains(function) { - return Err(anyhow!( - "file and extension access is not allowed in dashboard queries" - )); - } - } - - Ok(without_trailing_semicolon.to_string()) -} - -fn wrap_dashboard_query(sql: &str, limit: u32, chain_id: Option) -> String { - match chain_id { - Some(chain_id) => format!( - r#" - WITH contract_metadata AS ( - SELECT * - FROM contract_metadata_all - WHERE chain_id = {chain_id} - ) - SELECT * - FROM ({sql}) AS _blink_dashboard_query - LIMIT {limit} - "# - ), - None => format!("SELECT * FROM ({sql}) AS _blink_dashboard_query LIMIT {limit}"), - } -} - -fn value_ref_to_json(value: ValueRef<'_>) -> Value { - match value { - ValueRef::Null => Value::Null, - ValueRef::Boolean(value) => Value::Bool(value), - ValueRef::TinyInt(value) => Value::Number(Number::from(value)), - ValueRef::SmallInt(value) => Value::Number(Number::from(value)), - ValueRef::Int(value) => Value::Number(Number::from(value)), - ValueRef::BigInt(value) => Value::Number(Number::from(value)), - ValueRef::HugeInt(value) => i64::try_from(value) - .map(Number::from) - .map(Value::Number) - .unwrap_or_else(|_| Value::String(value.to_string())), - ValueRef::UTinyInt(value) => Value::Number(Number::from(value)), - ValueRef::USmallInt(value) => Value::Number(Number::from(value)), - ValueRef::UInt(value) => Value::Number(Number::from(value)), - ValueRef::UBigInt(value) => Value::Number(Number::from(value)), - ValueRef::Float(value) => Number::from_f64(value as f64) - .map(Value::Number) - .unwrap_or(Value::Null), - ValueRef::Double(value) => Number::from_f64(value) - .map(Value::Number) - .unwrap_or(Value::Null), - ValueRef::Decimal(value) => Value::String(value.to_string()), - ValueRef::Timestamp(unit, value) => { - Value::String(format!("{} {:?}", value, unit).to_ascii_lowercase()) - } - ValueRef::Text(value) => Value::String(String::from_utf8_lossy(value).into_owned()), - ValueRef::Blob(value) => Value::String(format!("0x{}", hex::encode(value))), - ValueRef::Date32(value) => Value::Number(Number::from(value)), - ValueRef::Time64(unit, value) => { - Value::String(format!("{} {:?}", value, unit).to_ascii_lowercase()) - } - ValueRef::Interval { - months, - days, - nanos, - } => Value::String(format!("{months} months {days} days {nanos} ns")), - other => Value::String(format!("{other:?}")), - } -} - -fn create_empty_metadata_current_view(conn: &Connection) -> Result<()> { - conn.execute_batch( - r#" - CREATE OR REPLACE TEMP VIEW bytecode_metadata_current AS - SELECT - CAST(NULL AS BLOB) AS contract_address, - CAST(NULL AS VARCHAR) AS language, - CAST(NULL AS VARCHAR) AS compiler_version, - CAST(false AS BOOLEAN) AS has_source_hash, - CAST(false AS BOOLEAN) AS is_erc20, - CAST(false AS BOOLEAN) AS is_erc721, - CAST(false AS BOOLEAN) AS is_erc1155, - CAST(false AS BOOLEAN) AS is_proxy_eip1967, - CAST(false AS BOOLEAN) AS is_proxy_minimal, - CAST(false AS BOOLEAN) AS uses_push0, - CAST(NULL AS TIMESTAMP) AS decoded_at - WHERE FALSE; - "#, - ) - .context("create empty metadata view") -} - -fn create_metadata_current_view(conn: &Connection) -> Result<()> { - let has_v1 = table_exists(conn, "bytecode_metadata")?; - let has_v2 = table_exists(conn, "bytecode_metadata_v2")?; - - if !has_v1 && !has_v2 { - return create_empty_metadata_current_view(conn); - } - - let address_meta = match (has_v1, has_v2) { - (false, false) => { - r#" - SELECT - CAST(NULL AS BLOB) AS contract_address, - CAST(NULL AS VARCHAR) AS language, - CAST(NULL AS VARCHAR) AS compiler_version, - CAST(false AS BOOLEAN) AS has_source_hash, - CAST(false AS BOOLEAN) AS is_erc20, - CAST(false AS BOOLEAN) AS is_erc721, - CAST(false AS BOOLEAN) AS is_erc1155, - CAST(false AS BOOLEAN) AS is_proxy_eip1967, - CAST(false AS BOOLEAN) AS is_proxy_minimal, - CAST(false AS BOOLEAN) AS uses_push0, - CAST(NULL AS TIMESTAMP) AS decoded_at - WHERE FALSE - "# - } - (true, false) => { - // v1 predates EIP-1167 detection — synthesize a false column so the - // view shape matches. - r#" - SELECT - contract_address, language, compiler_version, has_source_hash, - is_erc20, is_erc721, is_erc1155, is_proxy_eip1967, - CAST(false AS BOOLEAN) AS is_proxy_minimal, - uses_push0, CAST(NULL AS TIMESTAMP) AS decoded_at - FROM bytecode_metadata - "# - } - (false, true) => { - r#" - SELECT - contract_address, language, compiler_version, has_source_hash, - is_erc20, is_erc721, is_erc1155, is_proxy_eip1967, - is_proxy_minimal, uses_push0, decoded_at - FROM bytecode_metadata_v2 - "# - } - (true, true) => { - r#" - SELECT - contract_address, language, compiler_version, has_source_hash, - is_erc20, is_erc721, is_erc1155, is_proxy_eip1967, - is_proxy_minimal, uses_push0, decoded_at - FROM bytecode_metadata_v2 - UNION ALL - SELECT - v1.contract_address, v1.language, v1.compiler_version, - v1.has_source_hash, v1.is_erc20, v1.is_erc721, v1.is_erc1155, - v1.is_proxy_eip1967, CAST(false AS BOOLEAN) AS is_proxy_minimal, - v1.uses_push0, CAST(NULL AS TIMESTAMP) AS decoded_at - FROM bytecode_metadata v1 - WHERE NOT EXISTS ( - SELECT 1 - FROM bytecode_metadata_v2 v2 - WHERE v2.contract_address = v1.contract_address - ) - "# - } - }; - - let sql = format!( - r#" - CREATE OR REPLACE TEMP VIEW bytecode_metadata_current AS - {address_meta}; - "# - ); - conn.execute_batch(&sql) - .context("create combined metadata view") -} - -fn create_enrichment_current_view(conn: &Connection) -> Result<()> { - let sql = if table_exists(conn, "enrichment")? { - let chain_id = if column_exists(conn, "enrichment", "chain_id")? { - "chain_id" - } else { - "1::UBIGINT AS chain_id" - }; - let verification_source = if column_exists(conn, "enrichment", "verification_source")? { - "verification_source" - } else { - "CAST(NULL AS VARCHAR) AS verification_source" - }; - let match_type = if column_exists(conn, "enrichment", "match_type")? { - "match_type" - } else { - "CAST(NULL AS VARCHAR) AS match_type" - }; - let block_number = if column_exists(conn, "enrichment", "block_number")? { - "block_number" - } else { - "CAST(NULL AS UINTEGER) AS block_number" - }; - let create_index = if column_exists(conn, "enrichment", "create_index")? { - "create_index" - } else { - "CAST(NULL AS UINTEGER) AS create_index" - }; - format!( - r#" - CREATE OR REPLACE TEMP VIEW enrichment_current AS - SELECT - contract_address, - {chain_id}, - is_verified, - contract_name, - checked_at, - {verification_source}, - {match_type}, - {block_number}, - {create_index} - FROM enrichment; - "# - ) - } else { - r#" - CREATE OR REPLACE TEMP VIEW enrichment_current AS - SELECT - CAST(NULL AS BLOB) AS contract_address, - CAST(NULL AS UBIGINT) AS chain_id, - CAST(NULL AS BOOLEAN) AS is_verified, - CAST(NULL AS VARCHAR) AS contract_name, - CAST(NULL AS TIMESTAMP) AS checked_at, - CAST(NULL AS VARCHAR) AS verification_source, - CAST(NULL AS VARCHAR) AS match_type, - CAST(NULL AS UINTEGER) AS block_number, - CAST(NULL AS UINTEGER) AS create_index - WHERE FALSE; - "# - .to_string() - }; - conn.execute_batch(&sql) - .context("create enrichment compatibility view") -} - -fn verification_registry_loaded(conn: &Connection, chain_id: u64) -> Result { - if !table_exists(conn, "verification_registry_imports")? { - return Ok(false); - } - let count: i64 = conn - .query_row( - "SELECT COUNT(*) FROM verification_registry_imports WHERE source = 'verifier_alliance' AND chain_id = ?", - params![chain_id], - |row| row.get(0), - ) - .unwrap_or(0); - Ok(count > 0) -} - -fn create_standard_query_views(conn: &Connection, has_zellic: bool) -> Result<()> { - let has_hash = table_exists(conn, "bytecode_metadata_by_hash")?; - let has_counts = table_exists(conn, "zellic_bytecode_counts")?; - let hash_has_decoded_at = - has_hash && column_exists(conn, "bytecode_metadata_by_hash", "decoded_at")?; - - let bytecodes_sql = if has_zellic { - let count_join = if has_counts { - r#" - COALESCE(c.contract_count, 0)::UBIGINT AS contract_count - FROM zellic_bytecodes b - LEFT JOIN zellic_bytecode_counts c ON b.code_hash = c.code_hash - "# - } else { - r#" - CAST(NULL AS UBIGINT) AS contract_count - FROM zellic_bytecodes b - "# - }; - format!( - r#" - CREATE OR REPLACE TEMP VIEW bytecodes AS - SELECT - b.code_hash, - lower('0x' || hex(b.code_hash)) AS code_hash_hex, - b.n_code_bytes, - b.code, - {count_join}; - "# - ) - } else { - r#" - CREATE OR REPLACE TEMP VIEW bytecodes AS - SELECT - code_hash, - lower('0x' || hex(code_hash)) AS code_hash_hex, - any_value(n_code_bytes)::UINTEGER AS n_code_bytes, - any_value(code) AS code, - COUNT(*)::UBIGINT AS contract_count - FROM contracts - WHERE code_hash IS NOT NULL - GROUP BY code_hash; - "# - .to_string() - }; - conn.execute_batch(&bytecodes_sql) - .context("create bytecodes query view")?; - - let decoded_sql = if has_hash { - let decoded_at = if hash_has_decoded_at { - "decoded_at" - } else { - "CAST(NULL AS TIMESTAMP) AS decoded_at" - }; - let decoded_order = if hash_has_decoded_at { - "decoded_at DESC NULLS LAST" - } else { - "code_hash" - }; - format!( - r#" - CREATE OR REPLACE TEMP VIEW decoded_bytecodes AS - SELECT - code_hash, - lower('0x' || hex(code_hash)) AS code_hash_hex, - language, - compiler_version, - has_source_hash, - is_erc20, - is_erc721, - is_erc1155, - is_proxy_eip1967, - is_proxy_minimal, - uses_push0, - decoded_at - FROM ( - SELECT - code_hash, - language, - compiler_version, - has_source_hash, - is_erc20, - is_erc721, - is_erc1155, - is_proxy_eip1967, - is_proxy_minimal, - uses_push0, - {decoded_at}, - row_number() OVER ( - PARTITION BY code_hash - ORDER BY {decoded_order} - ) AS rn - FROM bytecode_metadata_by_hash - ) - WHERE rn = 1; - "# - ) - } else { - r#" - CREATE OR REPLACE TEMP VIEW decoded_bytecodes AS - SELECT - CAST(NULL AS BLOB) AS code_hash, - CAST(NULL AS VARCHAR) AS code_hash_hex, - CAST(NULL AS VARCHAR) AS language, - CAST(NULL AS VARCHAR) AS compiler_version, - CAST(false AS BOOLEAN) AS has_source_hash, - CAST(false AS BOOLEAN) AS is_erc20, - CAST(false AS BOOLEAN) AS is_erc721, - CAST(false AS BOOLEAN) AS is_erc1155, - CAST(false AS BOOLEAN) AS is_proxy_eip1967, - CAST(false AS BOOLEAN) AS is_proxy_minimal, - CAST(false AS BOOLEAN) AS uses_push0, - CAST(NULL AS TIMESTAMP) AS decoded_at - WHERE FALSE; - "# - .to_string() - }; - conn.execute_batch(&decoded_sql) - .context("create decoded bytecodes query view")?; - - let metadata_join = if has_zellic { - r#" - LEFT JOIN decoded_bytecodes m ON c.code_hash = m.code_hash - "# - } else { - r#" - LEFT JOIN bytecode_metadata_current m ON c.contract_address = m.contract_address - "# - }; - let is_verified_expr = if table_exists(conn, "verification_registry_imports")? { - "COALESCE(e.is_verified, false) AS is_verified" - } else { - "e.is_verified" - }; - let contract_metadata_sql = format!( - r#" - CREATE OR REPLACE TEMP VIEW contract_metadata_all AS - SELECT - c.chain_id, - c.block_number, - c.create_index, - c.contract_address, - lower('0x' || hex(c.contract_address)) AS address, - c.transaction_hash, - lower('0x' || hex(c.transaction_hash)) AS tx_hash, - c.block_hash, - lower('0x' || hex(c.block_hash)) AS block_hash_hex, - c.deployer, - lower('0x' || hex(c.deployer)) AS deployer_address, - c.factory, - lower('0x' || hex(c.factory)) AS factory_address, - c.code_hash, - lower('0x' || hex(c.code_hash)) AS code_hash_hex, - c.n_code_bytes, - m.language, - m.compiler_version, - COALESCE(m.has_source_hash, false) AS has_source_hash, - COALESCE(m.is_erc20, false) AS is_erc20, - COALESCE(m.is_erc721, false) AS is_erc721, - COALESCE(m.is_erc1155, false) AS is_erc1155, - COALESCE(m.is_proxy_eip1967, false) AS is_proxy_eip1967, - COALESCE(m.is_proxy_minimal, false) AS is_proxy_minimal, - COALESCE(m.uses_push0, false) AS uses_push0, - m.decoded_at, - {is_verified_expr}, - e.contract_name, - e.verification_source, - e.match_type, - e.checked_at AS verification_checked_at - FROM contracts c - {metadata_join} - LEFT JOIN enrichment_current e - ON c.contract_address = e.contract_address - AND c.chain_id = e.chain_id; - - CREATE OR REPLACE TEMP VIEW contract_metadata AS - SELECT * FROM contract_metadata_all; - "# - ); - conn.execute_batch(&contract_metadata_sql) - .context("create contract metadata query view")?; - - Ok(()) -} - -fn ensure_zellic_summary_tables(conn: &Connection) -> Result<()> { - if !table_exists(conn, "zellic_contracts")? { - return Ok(()); - } - - if !table_exists(conn, "zellic_bytecode_counts")? { - conn.execute_batch( - r#" - CREATE TABLE zellic_bytecode_counts AS - SELECT - bytecode_hash AS code_hash, - COUNT(*)::UBIGINT AS contract_count - FROM zellic_contracts - WHERE bytecode_hash IS NOT NULL - GROUP BY bytecode_hash; - "#, - ) - .context("create missing Zellic bytecode counts")?; - } - - if !table_exists(conn, "zellic_block_counts")? { - conn.execute_batch( - r#" - CREATE TABLE zellic_block_counts AS - SELECT - block_number, - COUNT(*)::UBIGINT AS contract_count - FROM zellic_contracts - WHERE block_number IS NOT NULL - GROUP BY block_number - ORDER BY block_number; - "#, - ) - .context("create missing Zellic block counts")?; - } - - Ok(()) -} - -fn backfill_enrichment_blocks(conn: &Connection) -> Result<()> { - if !table_exists(conn, "enrichment")? || !table_exists(conn, "zellic_contracts")? { - return Ok(()); - } - if !column_exists(conn, "enrichment", "block_number")? - || !column_exists(conn, "enrichment", "create_index")? - { - return Ok(()); - } - - conn.execute_batch( - r#" - UPDATE enrichment AS e - SET - block_number = z.block_number, - create_index = z.create_index - FROM zellic_contracts AS z - WHERE e.contract_address = z.contract_address - AND (e.block_number IS NULL OR e.create_index IS NULL); - "#, - ) - .context("backfill enrichment block positions")?; - Ok(()) -} - -impl Db { - pub fn open_with_mode(data_dir: &Path, contracts_glob: &str, read_only: bool) -> Result { - if !read_only { - std::fs::create_dir_all(data_dir) - .with_context(|| format!("create data dir {}", data_dir.display()))?; - } - let db_path = data_dir.join("blink.duckdb"); - - let conn = if read_only { - // Read-only connection coexists with an active writer since DuckDB - // only takes an exclusive lock on writers. - let config = Config::default() - .access_mode(AccessMode::ReadOnly) - .context("set read-only access mode")?; - Connection::open_with_flags(&db_path, config) - .with_context(|| format!("open duckdb (read-only) {}", db_path.display()))? - } else { - Connection::open(&db_path) - .with_context(|| format!("open duckdb {}", db_path.display()))? - }; - - if read_only { - // Schema is owned by the writer. If the tables aren't there yet - // (decode hasn't run), the queries will fail gracefully. - rebuild_contracts_view_for_conn(&conn, data_dir, contracts_glob)?; - return Ok(Self { - inner: Arc::new(Mutex::new(conn)), - data_dir: data_dir.to_path_buf(), - contracts_glob: contracts_glob.to_string(), - }); - } - - conn.execute_batch( - r#" - CREATE TABLE IF NOT EXISTS enrichment ( - contract_address BLOB, - chain_id UBIGINT DEFAULT 1, - is_verified BOOLEAN NOT NULL, - contract_name VARCHAR, - checked_at TIMESTAMP NOT NULL - ); - -- Track where each verification came from (verifier_alliance). - -- Added in a later migration; the IF NOT EXISTS guard keeps older - -- databases working without an explicit migration step. - ALTER TABLE enrichment ADD COLUMN IF NOT EXISTS chain_id UBIGINT DEFAULT 1; - ALTER TABLE enrichment ADD COLUMN IF NOT EXISTS verification_source VARCHAR; - ALTER TABLE enrichment ADD COLUMN IF NOT EXISTS match_type VARCHAR; - ALTER TABLE enrichment ADD COLUMN IF NOT EXISTS block_number UINTEGER; - ALTER TABLE enrichment ADD COLUMN IF NOT EXISTS create_index UINTEGER; - UPDATE enrichment SET chain_id = 1 WHERE chain_id IS NULL; - CREATE INDEX IF NOT EXISTS enrichment_chain_addr_idx ON enrichment(chain_id, contract_address); - CREATE INDEX IF NOT EXISTS enrichment_verified_idx ON enrichment(is_verified); - CREATE INDEX IF NOT EXISTS enrichment_source_idx ON enrichment(verification_source); - - CREATE TABLE IF NOT EXISTS bytecode_metadata_v2 ( - contract_address BLOB NOT NULL, - language VARCHAR, - compiler_version VARCHAR, - has_source_hash BOOLEAN NOT NULL, - is_erc20 BOOLEAN NOT NULL, - is_erc721 BOOLEAN NOT NULL, - is_erc1155 BOOLEAN NOT NULL, - is_proxy_eip1967 BOOLEAN NOT NULL, - is_proxy_minimal BOOLEAN NOT NULL DEFAULT false, - uses_push0 BOOLEAN NOT NULL, - source_file VARCHAR NOT NULL, - decoded_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP - ); - - CREATE TABLE IF NOT EXISTS bytecode_metadata_by_hash ( - code_hash BLOB NOT NULL, - language VARCHAR, - compiler_version VARCHAR, - has_source_hash BOOLEAN NOT NULL, - is_erc20 BOOLEAN NOT NULL, - is_erc721 BOOLEAN NOT NULL, - is_erc1155 BOOLEAN NOT NULL, - is_proxy_eip1967 BOOLEAN NOT NULL, - is_proxy_minimal BOOLEAN NOT NULL DEFAULT false, - uses_push0 BOOLEAN NOT NULL, - decoded_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP - ); - ALTER TABLE bytecode_metadata_by_hash - ADD COLUMN IF NOT EXISTS decoded_at TIMESTAMP; - -- EIP-1167 minimal proxy detection added later; backfill the column - -- with `false` on existing rows. DuckDB cannot add constrained - -- columns to an existing table. - ALTER TABLE bytecode_metadata_v2 - ADD COLUMN IF NOT EXISTS is_proxy_minimal BOOLEAN; - ALTER TABLE bytecode_metadata_by_hash - ADD COLUMN IF NOT EXISTS is_proxy_minimal BOOLEAN; - UPDATE bytecode_metadata_v2 - SET is_proxy_minimal = false - WHERE is_proxy_minimal IS NULL; - UPDATE bytecode_metadata_by_hash - SET is_proxy_minimal = false - WHERE is_proxy_minimal IS NULL; - "#, - ) - .context("create blink schema")?; - ensure_zellic_summary_tables(&conn)?; - backfill_enrichment_blocks(&conn)?; - let files = list_contract_parquet_files(data_dir, contracts_glob)?; - ensure_parquet_block_counts(&conn, &files)?; - rebuild_contracts_view_for_conn(&conn, data_dir, contracts_glob)?; - - Ok(Self { - inner: Arc::new(Mutex::new(conn)), - data_dir: data_dir.to_path_buf(), - contracts_glob: contracts_glob.to_string(), - }) - } - - fn rebuild_contracts_view_blocking(&self) -> Result<()> { - let conn = self.inner.blocking_lock(); - let files = list_contract_parquet_files(&self.data_dir, &self.contracts_glob)?; - ensure_parquet_block_counts(&conn, &files)?; - rebuild_contracts_view_for_conn(&conn, &self.data_dir, &self.contracts_glob) - } - - pub async fn refresh_contracts_view(&self) -> Result<()> { - let this = self.clone(); - tokio::task::spawn_blocking(move || this.rebuild_contracts_view_blocking()) - .await - .map_err(|e| anyhow!("join error: {}", e))? - } -} - -fn rebuild_contracts_view_for_conn( - conn: &Connection, - data_dir: &Path, - contracts_glob: &str, -) -> Result<()> { - let files = list_contract_parquet_files(data_dir, contracts_glob)?; - - // Use TEMP VIEWs so read-only mode (where the main database is locked - // for writes) can still set this up — temp views live in a session- - // scoped schema and don't require write access to the on-disk DB. - let has_zellic = - table_exists(conn, "zellic_contracts")? && table_exists(conn, "zellic_bytecodes")?; - - let empty_select = r#" - SELECT - CAST(NULL AS UINTEGER) AS block_number, - CAST(NULL AS BLOB) AS block_hash, - CAST(NULL AS UINTEGER) AS create_index, - CAST(NULL AS BLOB) AS transaction_hash, - CAST(NULL AS BLOB) AS contract_address, - CAST(NULL AS BLOB) AS deployer, - CAST(NULL AS BLOB) AS factory, - CAST(NULL AS BLOB) AS init_code, - CAST(NULL AS BLOB) AS code, - CAST(NULL AS BLOB) AS init_code_hash, - CAST(NULL AS UINTEGER) AS n_init_code_bytes, - CAST(NULL AS UINTEGER) AS n_code_bytes, - CAST(NULL AS BLOB) AS code_hash, - CAST(NULL AS UBIGINT) AS chain_id - WHERE FALSE - "#; - - let parquet_select = if files.is_empty() { - None - } else { - let list = files - .iter() - .map(|p| format!("'{}'", p.display().to_string().replace('\'', "''"))) - .collect::>() - .join(", "); - Some(format!( - r#" - SELECT - block_number, block_hash, create_index, transaction_hash, - contract_address, deployer, factory, init_code, code, - init_code_hash, n_init_code_bytes, n_code_bytes, - code_hash, chain_id - FROM read_parquet([{}], union_by_name = true) - "#, - list - )) - }; - - let zellic_select = if has_zellic { - Some( - r#" - SELECT - z.block_number, - CAST(NULL AS BLOB) AS block_hash, - z.create_index, - CAST(NULL AS BLOB) AS transaction_hash, - z.contract_address, - CAST(NULL AS BLOB) AS deployer, - CAST(NULL AS BLOB) AS factory, - CAST(NULL AS BLOB) AS init_code, - b.code, - CAST(NULL AS BLOB) AS init_code_hash, - CAST(NULL AS UINTEGER) AS n_init_code_bytes, - b.n_code_bytes, - z.bytecode_hash AS code_hash, - z.chain_id - FROM zellic_contracts z - LEFT JOIN zellic_bytecodes b ON z.bytecode_hash = b.code_hash - "# - .to_string(), - ) - } else { - None - }; - - let parquet_body = parquet_select - .clone() - .unwrap_or_else(|| empty_select.to_string()); - let parquet_sql = format!( - "CREATE OR REPLACE TEMP VIEW parquet_contracts AS\n{};", - parquet_body - ); - conn.execute_batch(&parquet_sql) - .with_context(|| format!("create parquet contracts view ({} files)", files.len()))?; - - let selects = [ - if files.is_empty() { - None - } else { - Some("SELECT * FROM parquet_contracts".to_string()) - }, - zellic_select, - ] - .into_iter() - .flatten() - .collect::>(); - let body = if selects.is_empty() { - empty_select.to_string() - } else { - selects.join("\nUNION ALL\n") - }; - let sql = format!("CREATE OR REPLACE TEMP VIEW contracts AS\n{};", body); - conn.execute_batch(&sql) - .with_context(|| format!("create contracts view ({} files)", files.len()))?; - create_metadata_current_view(conn)?; - create_enrichment_current_view(conn)?; - create_standard_query_views(conn, has_zellic)?; - Ok(()) -} - -impl Db { - pub async fn query_sql( - &self, - sql: String, - limit: u32, - chain_id: Option, - ) -> Result { - let inner = self.inner.clone(); - let normalized = normalize_read_only_sql(&sql)?; - let limit = limit.clamp(1, 1_000); - tokio::task::spawn_blocking(move || -> Result { - let started = Instant::now(); - let wrapped = wrap_dashboard_query(&normalized, limit, chain_id); - let conn = inner.blocking_lock(); - let mut stmt = conn.prepare(&wrapped).context("prepare dashboard query")?; - let mut rows = stmt.query([]).context("execute dashboard query")?; - let (columns, column_count) = { - let stmt = rows - .as_ref() - .context("dashboard query statement metadata unavailable")?; - (stmt.column_names(), stmt.column_count()) - }; - let mut out = Vec::new(); - while let Some(row) = rows.next().context("read dashboard query row")? { - let mut values = Vec::with_capacity(column_count); - for idx in 0..column_count { - values.push(value_ref_to_json(row.get_ref(idx)?)); - } - out.push(values); - } - Ok(SqlQueryResult { - columns, - row_count: out.len(), - rows: out, - limit, - elapsed_ms: started.elapsed().as_millis(), - }) - }) - .await - .map_err(|e| anyhow!("join error: {}", e))? - } - - pub async fn stats(&self, chain_id: u64) -> Result { - let inner = self.inner.clone(); - tokio::task::spawn_blocking(move || -> Result { - let conn = inner.blocking_lock(); - let (zellic_total, zellic_first, zellic_last): (i64, Option, Option) = - if chain_id == ETHEREUM_CHAIN_ID && table_exists(&conn, "zellic_block_counts")? { - conn.query_row( - r#" - SELECT - COALESCE(SUM(contract_count), 0)::BIGINT, - MIN(block_number), - MAX(block_number) - FROM zellic_block_counts - "#, - [], - |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), - ) - .unwrap_or((0, None, None)) - } else if chain_id == ETHEREUM_CHAIN_ID && table_exists(&conn, "zellic_contracts")? - { - conn.query_row( - "SELECT COUNT(*), MIN(block_number), MAX(block_number) FROM zellic_contracts", - [], - |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), - ) - .unwrap_or((0, None, None)) - } else { - (0, None, None) - }; - let (parquet_total, parquet_first, parquet_last): (i64, Option, Option) = - conn.query_row( - "SELECT COUNT(*), MIN(block_number), MAX(block_number) FROM parquet_contracts WHERE chain_id = ?", - params![chain_id], - |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), - ) - .unwrap_or((0, None, None)); - - let total = (zellic_total.max(0) + parquet_total.max(0)) as u64; - let first_block = [zellic_first, parquet_first] - .into_iter() - .flatten() - .min() - .unwrap_or(0) as u64; - let last_block = [zellic_last, parquet_last] - .into_iter() - .flatten() - .max() - .unwrap_or(0) as u64; - - let registry_loaded = verification_registry_loaded(&conn, chain_id)?; - let (enriched, verified): (i64, i64) = if registry_loaded { - let verified_zellic: i64 = - if chain_id == ETHEREUM_CHAIN_ID && table_exists(&conn, "zellic_contracts")? { - conn.query_row( - r#" - SELECT COUNT(*) - FROM zellic_contracts c - JOIN enrichment_current e - ON c.contract_address = e.contract_address - AND e.chain_id = ? - WHERE e.is_verified - "#, - params![chain_id], - |row| row.get(0), - ) - .unwrap_or(0) - } else { - 0 - }; - let verified_parquet: i64 = conn - .query_row( - r#" - SELECT COUNT(*) - FROM parquet_contracts c - JOIN enrichment_current e - ON c.contract_address = e.contract_address - AND c.chain_id = e.chain_id - WHERE e.is_verified - AND c.chain_id = ? - "#, - params![chain_id], - |row| row.get(0), - ) - .unwrap_or(0); - let verified = verified_zellic + verified_parquet; - (total as i64, verified) - } else { - conn.query_row( - "SELECT COUNT(*), COUNT(*) FILTER (WHERE is_verified) FROM enrichment_current WHERE chain_id = ?", - params![chain_id], - |row| Ok((row.get(0)?, row.get(1)?)), - ) - .unwrap_or((0, 0)) - }; - - let verified_count = verified.max(0) as u64; - let enriched_count = enriched.max(0) as u64; - let unverified_count = enriched_count.saturating_sub(verified_count); - let verified_pct = if enriched_count == 0 { - 0.0 - } else { - 100.0 * verified_count as f64 / enriched_count as f64 - }; - let enrichment_coverage_pct = if total == 0 { - 0.0 - } else { - 100.0 * enriched_count as f64 / total as f64 - }; - - Ok(Stats { - total_contracts: total, - verified_count, - unverified_count, - verified_pct, - last_block, - first_block, - enrichment_coverage_pct, - last_updated: Utc::now(), - }) - }) - .await - .map_err(|e| anyhow!("join error: {}", e))? - } - - pub async fn deploys_over_time( - &self, - chain_id: u64, - bucket_blocks: u64, - block_range: Option<(u64, u64)>, - ) -> Result> { - let inner = self.inner.clone(); - let data_dir = self.data_dir.clone(); - let contracts_glob = self.contracts_glob.clone(); - let bucket_blocks = bucket_blocks.max(1); - tokio::task::spawn_blocking(move || -> Result> { - let conn = inner.blocking_lock(); - let use_zellic_counts = - chain_id == ETHEREUM_CHAIN_ID && table_exists(&conn, "zellic_block_counts")?; - let use_zellic_contracts = - chain_id == ETHEREUM_CHAIN_ID && table_exists(&conn, "zellic_contracts")?; - let range_filter = block_range - .map(|(start, end)| format!(" AND block_number BETWEEN {start} AND {end}")) - .unwrap_or_default(); - let parquet_select = if table_exists(&conn, "parquet_block_counts")? { - format!( - r#" - SELECT - (block_number / {bucket_blocks})::UBIGINT AS bucket_id, - SUM(contract_count)::UBIGINT AS cnt - FROM parquet_block_counts - WHERE block_number IS NOT NULL - AND chain_id = {chain_id} - {range_filter} - GROUP BY bucket_id - "# - ) - } else { - let parquet_source = if block_range.is_some() { - let files = list_contract_parquet_files(&data_dir, &contracts_glob)?; - let files = - contract_parquet_files_for_block_range(&files, chain_id, block_range); - create_contract_parquet_view( - &conn, - "dashboard_range_parquet_contracts", - &files, - chain_id, - )?; - "dashboard_range_parquet_contracts" - } else { - "parquet_contracts" - }; - format!( - r#" - SELECT - (block_number / {bucket_blocks})::UBIGINT AS bucket_id, - COUNT(*)::UBIGINT AS cnt - FROM {parquet_source} - WHERE block_number IS NOT NULL - AND chain_id = {chain_id} - {range_filter} - GROUP BY bucket_id - "# - ) - }; - let zellic_select = if use_zellic_counts { - Some(format!( - r#" - SELECT - (block_number / {bucket_blocks})::UBIGINT AS bucket_id, - SUM(contract_count)::UBIGINT AS cnt - FROM zellic_block_counts - WHERE block_number IS NOT NULL - {range_filter} - GROUP BY bucket_id - "# - )) - } else if use_zellic_contracts { - Some(format!( - r#" - SELECT - (block_number / {bucket_blocks})::UBIGINT AS bucket_id, - COUNT(*)::UBIGINT AS cnt - FROM zellic_contracts - WHERE block_number IS NOT NULL - {range_filter} - GROUP BY bucket_id - "# - )) - } else { - None - }; - let sources = [zellic_select, Some(parquet_select)] - .into_iter() - .flatten() - .collect::>() - .join("\nUNION ALL\n"); - let sql = format!( - r#" - WITH bucket_counts AS ( - {sources} - ) - SELECT bucket_id, SUM(cnt)::UBIGINT AS cnt - FROM bucket_counts - GROUP BY bucket_id - ORDER BY bucket_id - "# - ); - let mut stmt = conn.prepare(&sql)?; - let rows = stmt.query_map([], read_u64_pair)?; - let mut out = Vec::new(); - for r in rows { - let (bucket_id, count) = r?; - let block_start = bucket_id * bucket_blocks; - let block_end = block_start + bucket_blocks - 1; - let mid = block_start + bucket_blocks / 2; - out.push(DeployBucket { - block_start, - block_end, - timestamp: crate::blocks::block_timestamp(chain_id, mid), - count, - }); - } - Ok(out) - }) - .await - .map_err(|e| anyhow!("join error: {}", e))? - } - - pub async fn verified_ratio_over_time( - &self, - chain_id: u64, - bucket_blocks: u64, - block_range: Option<(u64, u64)>, - ) -> Result> { - let inner = self.inner.clone(); - let data_dir = self.data_dir.clone(); - let contracts_glob = self.contracts_glob.clone(); - let bucket_blocks = bucket_blocks.max(1); - tokio::task::spawn_blocking(move || -> Result> { - let conn = inner.blocking_lock(); - let mut out = Vec::new(); - let registry_loaded = verification_registry_loaded(&conn, chain_id)?; - if let Some((start, end)) = block_range { - let parquet_total_select = if table_exists(&conn, "parquet_block_counts")? { - format!( - r#" - SELECT - (block_number / {bucket_blocks})::UBIGINT AS bucket_id, - SUM(contract_count)::UBIGINT AS total - FROM parquet_block_counts - WHERE block_number IS NOT NULL - AND chain_id = {chain_id} - AND block_number BETWEEN {start} AND {end} - GROUP BY bucket_id - "# - ) - } else { - let files = list_contract_parquet_files(&data_dir, &contracts_glob)?; - let files = - contract_parquet_files_for_block_range(&files, chain_id, block_range); - create_contract_parquet_view( - &conn, - "dashboard_range_parquet_contracts", - &files, - chain_id, - )?; - format!( - r#" - SELECT - (block_number / {bucket_blocks})::UBIGINT AS bucket_id, - COUNT(*)::UBIGINT AS total - FROM dashboard_range_parquet_contracts - WHERE block_number IS NOT NULL - AND chain_id = {chain_id} - AND block_number BETWEEN {start} AND {end} - GROUP BY bucket_id - "# - ) - }; - let zellic_total_select = if chain_id == ETHEREUM_CHAIN_ID - && table_exists(&conn, "zellic_block_counts")? - { - Some(format!( - r#" - SELECT - (block_number / {bucket_blocks})::UBIGINT AS bucket_id, - SUM(contract_count)::UBIGINT AS total - FROM zellic_block_counts - WHERE block_number IS NOT NULL - AND block_number BETWEEN {start} AND {end} - GROUP BY bucket_id - "# - )) - } else if chain_id == ETHEREUM_CHAIN_ID && table_exists(&conn, "zellic_contracts")? - { - Some(format!( - r#" - SELECT - (block_number / {bucket_blocks})::UBIGINT AS bucket_id, - COUNT(*)::UBIGINT AS total - FROM zellic_contracts - WHERE block_number IS NOT NULL - AND chain_id = {chain_id} - AND block_number BETWEEN {start} AND {end} - GROUP BY bucket_id - "# - )) - } else { - None - }; - let total_sources = [zellic_total_select, Some(parquet_total_select)] - .into_iter() - .flatten() - .collect::>() - .join("\nUNION ALL\n"); - let verified_value = "LEAST(COALESCE(checked.verified, 0), totals.total)"; - let (unverified_expr, unknown_expr) = if registry_loaded { - ( - format!("GREATEST(totals.total - {verified_value}, 0)::UBIGINT"), - "0::UBIGINT".to_string(), - ) - } else { - ( - "COALESCE(checked.unverified, 0)::UBIGINT".to_string(), - format!( - "GREATEST(totals.total - {verified_value} - COALESCE(checked.unverified, 0), 0)::UBIGINT" - ), - ) - }; - let sql = format!( - r#" - WITH total_rows AS ( - {total_sources} - ), - totals AS ( - SELECT bucket_id, SUM(total)::UBIGINT AS total - FROM total_rows - GROUP BY bucket_id - ), - checked AS ( - SELECT - (block_number / {bucket_blocks})::UBIGINT AS bucket_id, - COUNT(*) FILTER (WHERE is_verified)::UBIGINT AS verified, - COUNT(*) FILTER (WHERE is_verified IS FALSE)::UBIGINT AS unverified - FROM enrichment_current - WHERE block_number IS NOT NULL - AND chain_id = {chain_id} - AND block_number BETWEEN {start} AND {end} - GROUP BY bucket_id - ) - SELECT - totals.bucket_id, - {verified_value}::UBIGINT AS verified, - {unverified_expr} AS unverified, - {unknown_expr} AS unknown - FROM totals - LEFT JOIN checked ON totals.bucket_id = checked.bucket_id - ORDER BY totals.bucket_id - "# - ); - let mut stmt = conn.prepare(&sql)?; - let rows = stmt.query_map([], read_verified_bucket_row)?; - for r in rows { - let (bucket_id, verified, unverified, unknown) = r?; - let block_start = bucket_id * bucket_blocks; - let block_end = block_start + bucket_blocks - 1; - let mid = block_start + bucket_blocks / 2; - out.push(VerifiedRatioBucket { - block_start, - block_end, - timestamp: crate::blocks::block_timestamp(chain_id, mid), - verified, - unverified, - unknown, - }); - } - return Ok(out); - } - if chain_id == ETHEREUM_CHAIN_ID && table_exists(&conn, "zellic_contracts")? { - if !table_exists(&conn, "zellic_block_counts")? { - return Ok(out); - } - - let sql = match (registry_loaded, block_range.is_some()) { - (true, false) => r#" - WITH totals AS ( - SELECT - (block_number / ?)::UBIGINT AS bucket_id, - SUM(contract_count)::UBIGINT AS total - FROM zellic_block_counts - WHERE block_number IS NOT NULL - GROUP BY bucket_id - ), - checked AS ( - SELECT - (z.block_number / ?)::UBIGINT AS bucket_id, - COUNT(*)::UBIGINT AS verified - FROM zellic_contracts z - JOIN enrichment_current e - ON z.contract_address = e.contract_address - AND e.chain_id = ? - WHERE z.block_number IS NOT NULL - AND e.is_verified - GROUP BY bucket_id - ) - SELECT - totals.bucket_id, - COALESCE(checked.verified, 0)::UBIGINT AS verified, - GREATEST(totals.total - COALESCE(checked.verified, 0), 0)::UBIGINT AS unverified, - 0::UBIGINT AS unknown - FROM totals - LEFT JOIN checked ON totals.bucket_id = checked.bucket_id - ORDER BY totals.bucket_id - "#, - (true, true) => r#" - WITH totals AS ( - SELECT - (block_number / ?)::UBIGINT AS bucket_id, - SUM(contract_count)::UBIGINT AS total - FROM zellic_block_counts - WHERE block_number IS NOT NULL - AND block_number BETWEEN ? AND ? - GROUP BY bucket_id - ), - checked AS ( - SELECT - (z.block_number / ?)::UBIGINT AS bucket_id, - COUNT(*)::UBIGINT AS verified - FROM zellic_contracts z - JOIN enrichment_current e - ON z.contract_address = e.contract_address - AND e.chain_id = ? - WHERE z.block_number IS NOT NULL - AND z.block_number BETWEEN ? AND ? - AND e.is_verified - GROUP BY bucket_id - ) - SELECT - totals.bucket_id, - COALESCE(checked.verified, 0)::UBIGINT AS verified, - GREATEST(totals.total - COALESCE(checked.verified, 0), 0)::UBIGINT AS unverified, - 0::UBIGINT AS unknown - FROM totals - LEFT JOIN checked ON totals.bucket_id = checked.bucket_id - ORDER BY totals.bucket_id - "#, - (false, false) => r#" - WITH totals AS ( - SELECT - (block_number / ?)::UBIGINT AS bucket_id, - SUM(contract_count)::UBIGINT AS total - FROM zellic_block_counts - WHERE block_number IS NOT NULL - GROUP BY bucket_id - ), - checked AS ( - SELECT - (block_number / ?)::UBIGINT AS bucket_id, - COUNT(*) FILTER (WHERE is_verified)::UBIGINT AS verified, - COUNT(*) FILTER (WHERE is_verified IS FALSE)::UBIGINT AS unverified - FROM enrichment_current - WHERE block_number IS NOT NULL - AND chain_id = ? - GROUP BY bucket_id - ) - SELECT - totals.bucket_id, - COALESCE(checked.verified, 0)::UBIGINT AS verified, - COALESCE(checked.unverified, 0)::UBIGINT AS unverified, - GREATEST( - totals.total - - COALESCE(checked.verified, 0) - - COALESCE(checked.unverified, 0), - 0 - )::UBIGINT AS unknown - FROM totals - LEFT JOIN checked ON totals.bucket_id = checked.bucket_id - ORDER BY totals.bucket_id - "#, - (false, true) => r#" - WITH totals AS ( - SELECT - (block_number / ?)::UBIGINT AS bucket_id, - SUM(contract_count)::UBIGINT AS total - FROM zellic_block_counts - WHERE block_number IS NOT NULL - AND block_number BETWEEN ? AND ? - GROUP BY bucket_id - ), - checked AS ( - SELECT - (block_number / ?)::UBIGINT AS bucket_id, - COUNT(*) FILTER (WHERE is_verified)::UBIGINT AS verified, - COUNT(*) FILTER (WHERE is_verified IS FALSE)::UBIGINT AS unverified - FROM enrichment_current - WHERE block_number IS NOT NULL - AND chain_id = ? - AND block_number BETWEEN ? AND ? - GROUP BY bucket_id - ) - SELECT - totals.bucket_id, - COALESCE(checked.verified, 0)::UBIGINT AS verified, - COALESCE(checked.unverified, 0)::UBIGINT AS unverified, - GREATEST( - totals.total - - COALESCE(checked.verified, 0) - - COALESCE(checked.unverified, 0), - 0 - )::UBIGINT AS unknown - FROM totals - LEFT JOIN checked ON totals.bucket_id = checked.bucket_id - ORDER BY totals.bucket_id - "#, - }; - let mut stmt = conn.prepare(sql)?; - let rows = match block_range { - None => stmt.query_map( - params![bucket_blocks, bucket_blocks, chain_id], - read_verified_bucket_row, - )?, - Some((start, end)) => stmt.query_map( - params![bucket_blocks, start, end, bucket_blocks, chain_id, start, end], - read_verified_bucket_row, - )?, - }; - for r in rows { - let (bucket_id, verified, unverified, unknown) = r?; - let block_start = bucket_id * bucket_blocks; - let block_end = block_start + bucket_blocks - 1; - let mid = block_start + bucket_blocks / 2; - out.push(VerifiedRatioBucket { - block_start, - block_end, - timestamp: crate::blocks::block_timestamp(chain_id, mid), - verified, - unverified, - unknown, - }); - } - return Ok(out); - } - - let sql = match (registry_loaded, block_range.is_some()) { - (true, false) => r#" - SELECT - (c.block_number / ?)::UBIGINT AS bucket_id, - COUNT(*) FILTER (WHERE e.is_verified)::UBIGINT AS verified, - COUNT(*) FILTER (WHERE e.is_verified IS NULL OR e.is_verified IS FALSE)::UBIGINT AS unverified, - 0::UBIGINT AS unknown - FROM contracts c - LEFT JOIN enrichment_current e - ON c.contract_address = e.contract_address - AND c.chain_id = e.chain_id - WHERE c.block_number IS NOT NULL - AND c.chain_id = ? - GROUP BY bucket_id - ORDER BY bucket_id - "#, - (true, true) => r#" - SELECT - (c.block_number / ?)::UBIGINT AS bucket_id, - COUNT(*) FILTER (WHERE e.is_verified)::UBIGINT AS verified, - COUNT(*) FILTER (WHERE e.is_verified IS NULL OR e.is_verified IS FALSE)::UBIGINT AS unverified, - 0::UBIGINT AS unknown - FROM contracts c - LEFT JOIN enrichment_current e - ON c.contract_address = e.contract_address - AND c.chain_id = e.chain_id - WHERE c.block_number IS NOT NULL - AND c.chain_id = ? - AND c.block_number BETWEEN ? AND ? - GROUP BY bucket_id - ORDER BY bucket_id - "#, - (false, false) => r#" - SELECT - (c.block_number / ?)::UBIGINT AS bucket_id, - COUNT(*) FILTER (WHERE e.is_verified)::UBIGINT AS verified, - COUNT(*) FILTER (WHERE e.is_verified IS FALSE)::UBIGINT AS unverified, - COUNT(*) FILTER (WHERE e.is_verified IS NULL)::UBIGINT AS unknown - FROM contracts c - LEFT JOIN enrichment_current e - ON c.contract_address = e.contract_address - AND c.chain_id = e.chain_id - WHERE c.block_number IS NOT NULL - AND c.chain_id = ? - GROUP BY bucket_id - ORDER BY bucket_id - "#, - (false, true) => r#" - SELECT - (c.block_number / ?)::UBIGINT AS bucket_id, - COUNT(*) FILTER (WHERE e.is_verified)::UBIGINT AS verified, - COUNT(*) FILTER (WHERE e.is_verified IS FALSE)::UBIGINT AS unverified, - COUNT(*) FILTER (WHERE e.is_verified IS NULL)::UBIGINT AS unknown - FROM contracts c - LEFT JOIN enrichment_current e - ON c.contract_address = e.contract_address - AND c.chain_id = e.chain_id - WHERE c.block_number IS NOT NULL - AND c.chain_id = ? - AND c.block_number BETWEEN ? AND ? - GROUP BY bucket_id - ORDER BY bucket_id - "#, - }; - let mut stmt = conn.prepare(sql)?; - let rows = match block_range { - None => stmt.query_map( - params![bucket_blocks, chain_id], - read_verified_bucket_row, - )?, - Some((start, end)) => stmt.query_map( - params![bucket_blocks, chain_id, start, end], - read_verified_bucket_row, - )?, - }; - for r in rows { - let (bucket_id, verified, unverified, unknown) = r?; - let block_start = bucket_id * bucket_blocks; - let block_end = block_start + bucket_blocks - 1; - let mid = block_start + bucket_blocks / 2; - out.push(VerifiedRatioBucket { - block_start, - block_end, - timestamp: crate::blocks::block_timestamp(chain_id, mid), - verified, - unverified, - unknown, - }); - } - Ok(out) - }) - .await - .map_err(|e| anyhow!("join error: {}", e))? - } - - pub async fn bytecode_size_distribution(&self, chain_id: u64) -> Result> { - let inner = self.inner.clone(); - tokio::task::spawn_blocking(move || -> Result> { - let conn = inner.blocking_lock(); - - // Fixed semantic buckets keep the heavy tail readable and stop the - // first 1KB range from hiding zero-byte and minimal-proxy contracts. - let bucket_defs: [(u64, u64, &str); 12] = [ - (0, 0, "0 B"), - (1, 32, "1-32 B"), - (33, 44, "33-44 B"), - (45, 45, "45 B minimal proxy"), - (45, 45, "45 B other"), - (46, 64, "46-64 B"), - (65, 256, "65-256 B"), - (257, 1_024, "257 B-1 KB"), - (1_025, 4_096, "1-4 KB"), - (4_097, 8_192, "4-8 KB"), - (8_193, 16_384, "8-16 KB"), - (16_385, 24_576, "16-24 KB"), - ]; - let mut counts = vec![0u64; bucket_defs.len()]; - let use_zellic = chain_id == ETHEREUM_CHAIN_ID - && table_exists(&conn, "zellic_bytecodes")? - && table_exists(&conn, "zellic_bytecode_counts")?; - let sql = if use_zellic { - r#" - SELECT - CASE - WHEN b.n_code_bytes = 0 THEN 0 - WHEN b.n_code_bytes <= 32 THEN 1 - WHEN b.n_code_bytes <= 44 THEN 2 - WHEN b.n_code_bytes = 45 AND COALESCE(m.is_proxy_minimal, false) THEN 3 - WHEN b.n_code_bytes = 45 THEN 4 - WHEN b.n_code_bytes <= 64 THEN 5 - WHEN b.n_code_bytes <= 256 THEN 6 - WHEN b.n_code_bytes <= 1024 THEN 7 - WHEN b.n_code_bytes <= 4096 THEN 8 - WHEN b.n_code_bytes <= 8192 THEN 9 - WHEN b.n_code_bytes <= 16384 THEN 10 - ELSE 11 - END AS bin_id, - SUM(c.contract_count)::UBIGINT AS cnt - FROM zellic_bytecodes b - JOIN zellic_bytecode_counts c ON b.code_hash = c.code_hash - LEFT JOIN bytecode_metadata_by_hash m ON b.code_hash = m.code_hash - WHERE b.n_code_bytes IS NOT NULL - AND b.n_code_bytes <= 24576 - GROUP BY bin_id - ORDER BY bin_id - "# - .to_string() - } else { - r#" - SELECT - CASE - WHEN c.n_code_bytes = 0 THEN 0 - WHEN c.n_code_bytes <= 32 THEN 1 - WHEN c.n_code_bytes <= 44 THEN 2 - WHEN c.n_code_bytes = 45 AND COALESCE(m.is_proxy_minimal, false) THEN 3 - WHEN c.n_code_bytes = 45 THEN 4 - WHEN c.n_code_bytes <= 64 THEN 5 - WHEN c.n_code_bytes <= 256 THEN 6 - WHEN c.n_code_bytes <= 1024 THEN 7 - WHEN c.n_code_bytes <= 4096 THEN 8 - WHEN c.n_code_bytes <= 8192 THEN 9 - WHEN c.n_code_bytes <= 16384 THEN 10 - ELSE 11 - END AS bin_id, - COUNT(*)::UBIGINT AS cnt - FROM contracts c - LEFT JOIN bytecode_metadata_by_hash m ON c.code_hash = m.code_hash - WHERE c.n_code_bytes IS NOT NULL - AND c.n_code_bytes <= 24576 - AND c.chain_id = ? - GROUP BY bin_id - ORDER BY bin_id - "# - .to_string() - }; - let mut stmt = conn.prepare(&sql)?; - let rows = if use_zellic { - stmt.query_map([], read_u64_pair)? - } else { - stmt.query_map(params![chain_id], read_u64_pair)? - }; - for r in rows { - let (bin_id, count) = r?; - if let Some(slot) = counts.get_mut(bin_id as usize) { - *slot = count; - } - } - Ok(bucket_defs - .into_iter() - .enumerate() - .map(|(i, (size_min, size_max, label))| SizeBin { - label: label.to_string(), - size_min, - size_max, - count: counts[i], - }) - .collect()) - }) - .await - .map_err(|e| anyhow!("join error: {}", e))? - } - - pub async fn top_compilers(&self, chain_id: u64, limit: u32) -> Result> { - let inner = self.inner.clone(); - let limit = limit.clamp(1, 50); - tokio::task::spawn_blocking(move || -> Result> { - let conn = inner.blocking_lock(); - // Compiler distribution is bytecode-derived. Verification sources - // can confirm source publication, but local decode remains the - // source of truth for compiler metadata in this dashboard. - let use_zellic = chain_id == ETHEREUM_CHAIN_ID - && table_exists(&conn, "zellic_bytecode_counts")? - && table_exists(&conn, "bytecode_metadata_by_hash")?; - let sql = if use_zellic { - r#" - SELECT m.compiler_version, SUM(c.contract_count)::UBIGINT AS cnt - FROM bytecode_metadata_by_hash m - JOIN zellic_bytecode_counts c ON m.code_hash = c.code_hash - WHERE m.compiler_version IS NOT NULL - GROUP BY m.compiler_version - ORDER BY cnt DESC - LIMIT ? - "# - } else { - r#" - WITH counts AS ( - SELECT code_hash, COUNT(*)::UBIGINT AS contract_count - FROM contracts - WHERE chain_id = ? - AND code_hash IS NOT NULL - GROUP BY code_hash - ) - SELECT m.compiler_version, SUM(c.contract_count)::UBIGINT AS cnt - FROM counts c - JOIN bytecode_metadata_by_hash m ON c.code_hash = m.code_hash - WHERE m.compiler_version IS NOT NULL - GROUP BY m.compiler_version - ORDER BY cnt DESC - LIMIT ? - "# - }; - let mut stmt = conn.prepare(sql)?; - let rows = if use_zellic { - stmt.query_map(params![limit as i64], read_string_u64_pair)? - } else { - stmt.query_map(params![chain_id, limit as i64], read_string_u64_pair)? - }; - let mut out = Vec::new(); - for r in rows { - let (compiler_version, count) = r?; - out.push(CompilerCount { - compiler_version, - count, - }); - } - Ok(out) - }) - .await - .map_err(|e| anyhow!("join error: {}", e))? - } - - pub async fn compiler_version_total(&self, chain_id: u64) -> Result { - let inner = self.inner.clone(); - tokio::task::spawn_blocking(move || -> Result { - let conn = inner.blocking_lock(); - if chain_id == ETHEREUM_CHAIN_ID - && table_exists(&conn, "zellic_bytecode_counts")? - && table_exists(&conn, "bytecode_metadata_by_hash")? - { - let count: i64 = conn - .query_row( - r#" - SELECT COALESCE(SUM(c.contract_count), 0)::BIGINT - FROM bytecode_metadata_by_hash m - JOIN zellic_bytecode_counts c ON m.code_hash = c.code_hash - WHERE m.compiler_version IS NOT NULL - "#, - [], - |row| row.get(0), - ) - .unwrap_or(0); - return Ok(count.max(0) as u64); - } - let count: i64 = conn - .query_row( - r#" - WITH counts AS ( - SELECT code_hash, COUNT(*)::UBIGINT AS contract_count - FROM contracts - WHERE chain_id = ? - AND code_hash IS NOT NULL - GROUP BY code_hash - ) - SELECT COALESCE(SUM(c.contract_count), 0)::BIGINT - FROM counts c - JOIN bytecode_metadata_by_hash m ON c.code_hash = m.code_hash - WHERE m.compiler_version IS NOT NULL - "#, - params![chain_id], - |row| row.get(0), - ) - .unwrap_or(0); - Ok(count.max(0) as u64) - }) - .await - .map_err(|e| anyhow!("join error: {}", e))? - } - - pub async fn language_distribution(&self, chain_id: u64) -> Result> { - let inner = self.inner.clone(); - tokio::task::spawn_blocking(move || -> Result> { - let conn = inner.blocking_lock(); - let use_zellic = chain_id == ETHEREUM_CHAIN_ID - && table_exists(&conn, "zellic_bytecode_counts")? - && table_exists(&conn, "bytecode_metadata_by_hash")?; - let sql = if use_zellic { - r#" - SELECT COALESCE(m.language, 'unknown') AS lang, - SUM(c.contract_count)::UBIGINT AS cnt - FROM zellic_bytecode_counts c - LEFT JOIN bytecode_metadata_by_hash m ON c.code_hash = m.code_hash - GROUP BY lang - ORDER BY cnt DESC - "# - } else { - r#" - WITH counts AS ( - SELECT code_hash, COUNT(*)::UBIGINT AS contract_count - FROM contracts - WHERE chain_id = ? - AND code_hash IS NOT NULL - GROUP BY code_hash - ) - SELECT COALESCE(m.language, 'unknown') AS lang, - SUM(c.contract_count)::UBIGINT AS cnt - FROM counts c - LEFT JOIN bytecode_metadata_by_hash m ON c.code_hash = m.code_hash - GROUP BY lang - ORDER BY cnt DESC - "# - }; - let mut stmt = conn.prepare(sql)?; - let rows = if use_zellic { - stmt.query_map([], read_string_u64_pair)? - } else { - stmt.query_map(params![chain_id], read_string_u64_pair)? - }; - let mut out = Vec::new(); - for r in rows { - let (language, count) = r?; - out.push(LanguageCount { language, count }); - } - Ok(out) - }) - .await - .map_err(|e| anyhow!("join error: {}", e))? - } - - pub async fn standards_breakdown(&self, chain_id: u64) -> Result { - let inner = self.inner.clone(); - tokio::task::spawn_blocking(move || -> Result { - let conn = inner.blocking_lock(); - if chain_id == ETHEREUM_CHAIN_ID - && table_exists(&conn, "zellic_bytecode_counts")? - && table_exists(&conn, "bytecode_metadata_by_hash")? - { - return conn - .query_row( - r#" - SELECT - COALESCE(SUM(CASE WHEN m.is_erc20 THEN c.contract_count ELSE 0 END), 0)::UBIGINT, - COALESCE(SUM(CASE WHEN m.is_erc721 THEN c.contract_count ELSE 0 END), 0)::UBIGINT, - COALESCE(SUM(CASE WHEN m.is_erc1155 THEN c.contract_count ELSE 0 END), 0)::UBIGINT, - COALESCE(SUM(CASE WHEN m.is_proxy_eip1967 THEN c.contract_count ELSE 0 END), 0)::UBIGINT, - COALESCE(SUM(CASE WHEN m.is_proxy_minimal THEN c.contract_count ELSE 0 END), 0)::UBIGINT, - COALESCE(SUM(CASE WHEN m.uses_push0 THEN c.contract_count ELSE 0 END), 0)::UBIGINT, - COALESCE(SUM(CASE WHEN m.has_source_hash THEN c.contract_count ELSE 0 END), 0)::UBIGINT, - COALESCE(SUM(CASE WHEN m.code_hash IS NOT NULL THEN c.contract_count ELSE 0 END), 0)::UBIGINT - FROM zellic_bytecode_counts c - LEFT JOIN bytecode_metadata_by_hash m ON c.code_hash = m.code_hash - "#, - [], - |row| { - Ok(StandardsBreakdown { - erc20: row.get(0)?, - erc721: row.get(1)?, - erc1155: row.get(2)?, - proxy_eip1967: row.get(3)?, - proxy_minimal: row.get(4)?, - uses_push0: row.get(5)?, - has_source_hash: row.get(6)?, - total_decoded: row.get(7)?, - }) - }, - ) - .or_else(|_| { - Ok(StandardsBreakdown { - erc20: 0, - erc721: 0, - erc1155: 0, - proxy_eip1967: 0, - proxy_minimal: 0, - uses_push0: 0, - has_source_hash: 0, - total_decoded: 0, - }) - }); - } - conn.query_row( - r#" - WITH counts AS ( - SELECT code_hash, COUNT(*)::UBIGINT AS contract_count - FROM contracts - WHERE chain_id = ? - AND code_hash IS NOT NULL - GROUP BY code_hash - ) - SELECT - COALESCE(SUM(CASE WHEN m.is_erc20 THEN c.contract_count ELSE 0 END), 0)::UBIGINT, - COALESCE(SUM(CASE WHEN m.is_erc721 THEN c.contract_count ELSE 0 END), 0)::UBIGINT, - COALESCE(SUM(CASE WHEN m.is_erc1155 THEN c.contract_count ELSE 0 END), 0)::UBIGINT, - COALESCE(SUM(CASE WHEN m.is_proxy_eip1967 THEN c.contract_count ELSE 0 END), 0)::UBIGINT, - COALESCE(SUM(CASE WHEN m.is_proxy_minimal THEN c.contract_count ELSE 0 END), 0)::UBIGINT, - COALESCE(SUM(CASE WHEN m.uses_push0 THEN c.contract_count ELSE 0 END), 0)::UBIGINT, - COALESCE(SUM(CASE WHEN m.has_source_hash THEN c.contract_count ELSE 0 END), 0)::UBIGINT, - COALESCE(SUM(CASE WHEN m.code_hash IS NOT NULL THEN c.contract_count ELSE 0 END), 0)::UBIGINT - FROM counts c - LEFT JOIN bytecode_metadata_by_hash m ON c.code_hash = m.code_hash - "#, - params![chain_id], - |row| { - Ok(StandardsBreakdown { - erc20: row.get(0)?, - erc721: row.get(1)?, - erc1155: row.get(2)?, - proxy_eip1967: row.get(3)?, - proxy_minimal: row.get(4)?, - uses_push0: row.get(5)?, - has_source_hash: row.get(6)?, - total_decoded: row.get(7)?, - }) - }, - ) - .or_else(|_| { - Ok(StandardsBreakdown { - erc20: 0, - erc721: 0, - erc1155: 0, - proxy_eip1967: 0, - proxy_minimal: 0, - uses_push0: 0, - has_source_hash: 0, - total_decoded: 0, - }) - }) - }) - .await - .map_err(|e| anyhow!("join error: {}", e))? - } - - pub async fn recent_contracts( - &self, - chain_id: u64, - limit: u32, - cursor: Option, - ) -> Result { - let inner = self.inner.clone(); - let data_dir = self.data_dir.clone(); - let contracts_glob = self.contracts_glob.clone(); - let limit = limit.clamp(1, 200); - let (mut contracts, may_have_more_external) = - tokio::task::spawn_blocking(move || -> Result<(Vec, bool)> { - let conn = inner.blocking_lock(); - let page_limit = limit as i64 + 1; - let has_zellic = chain_id == ETHEREUM_CHAIN_ID - && table_exists(&conn, "zellic_contracts")? - && table_exists(&conn, "zellic_bytecodes")?; - let has_hash_metadata = table_exists(&conn, "bytecode_metadata_by_hash")?; - let parquet_files = list_contract_parquet_files(&data_dir, &contracts_glob)?; - let max_contracts_block = - max_contract_file_block_for_chain(&parquet_files, chain_id); - let max_zellic_block: Option = if chain_id == ETHEREUM_CHAIN_ID && has_zellic { - let block: Option = conn - .query_row( - "SELECT MAX(block_number) FROM zellic_contracts", - [], - |row| row.get(0), - ) - .unwrap_or(None); - block.map(u64::from) - } else { - None - }; - let has_external_contracts = max_contracts_block > max_zellic_block; - let use_external_recent = has_external_contracts - && cursor - .map(|cursor| cursor.block_number > max_zellic_block.unwrap_or(0)) - .unwrap_or(true); - if use_external_recent { - let recent_files = - recent_contract_parquet_files(&parquet_files, chain_id, cursor); - create_recent_parquet_contracts_view(&conn, &recent_files, chain_id)?; - } - conn.execute_batch(&format!( - r#" - CREATE OR REPLACE TEMP VIEW recent_enrichment AS - SELECT * - FROM enrichment_current - WHERE chain_id = {}; - "#, - chain_id - )) - .context("create recent enrichment view")?; - conn.execute_batch(&format!( - r#" - CREATE OR REPLACE TEMP VIEW recent_contracts_all AS - SELECT * - FROM contracts - WHERE chain_id = {}; - "#, - chain_id - )) - .context("create recent contracts view")?; - let registry_loaded = verification_registry_loaded(&conn, chain_id)?; - - // For Zellic imports, compiler metadata is keyed by bytecode hash. - // Select the tiny recent page first, then join hash metadata; going - // through bytecode_metadata_current would expand hash metadata back - // to tens of millions of contract-address rows for every page. - let sql = if use_external_recent && has_hash_metadata && cursor.is_some() { - r#" - WITH page AS ( - SELECT - c.contract_address, - c.block_number, - c.create_index, - c.deployer, - c.n_code_bytes, - c.code_hash - FROM recent_parquet_contracts c - WHERE c.block_number IS NOT NULL - AND ( - c.block_number < ? - OR (c.block_number = ? AND c.create_index < ?) - ) - ORDER BY c.block_number DESC, c.create_index DESC - LIMIT ? - ) - SELECT - page.contract_address, - page.block_number, - page.create_index, - page.deployer, - page.n_code_bytes, - e.is_verified, - e.contract_name, - h.compiler_version - FROM page - LEFT JOIN recent_enrichment e ON page.contract_address = e.contract_address - LEFT JOIN bytecode_metadata_by_hash h ON page.code_hash = h.code_hash - ORDER BY page.block_number DESC, page.create_index DESC - "# - } else if use_external_recent && has_hash_metadata { - r#" - WITH page AS ( - SELECT - c.contract_address, - c.block_number, - c.create_index, - c.deployer, - c.n_code_bytes, - c.code_hash - FROM recent_parquet_contracts c - WHERE c.block_number IS NOT NULL - ORDER BY c.block_number DESC, c.create_index DESC - LIMIT ? - ) - SELECT - page.contract_address, - page.block_number, - page.create_index, - page.deployer, - page.n_code_bytes, - e.is_verified, - e.contract_name, - h.compiler_version - FROM page - LEFT JOIN recent_enrichment e ON page.contract_address = e.contract_address - LEFT JOIN bytecode_metadata_by_hash h ON page.code_hash = h.code_hash - ORDER BY page.block_number DESC, page.create_index DESC - "# - } else if use_external_recent && cursor.is_some() { - r#" - WITH page AS ( - SELECT - c.contract_address, - c.block_number, - c.create_index, - c.deployer, - c.n_code_bytes - FROM recent_parquet_contracts c - WHERE c.block_number IS NOT NULL - AND ( - c.block_number < ? - OR (c.block_number = ? AND c.create_index < ?) - ) - ORDER BY c.block_number DESC, c.create_index DESC - LIMIT ? - ) - SELECT - page.contract_address, - page.block_number, - page.create_index, - page.deployer, - page.n_code_bytes, - e.is_verified, - e.contract_name, - CAST(NULL AS VARCHAR) AS compiler_version - FROM page - LEFT JOIN recent_enrichment e ON page.contract_address = e.contract_address - ORDER BY page.block_number DESC, page.create_index DESC - "# - } else if use_external_recent { - r#" - WITH page AS ( - SELECT - c.contract_address, - c.block_number, - c.create_index, - c.deployer, - c.n_code_bytes - FROM recent_parquet_contracts c - WHERE c.block_number IS NOT NULL - ORDER BY c.block_number DESC, c.create_index DESC - LIMIT ? - ) - SELECT - page.contract_address, - page.block_number, - page.create_index, - page.deployer, - page.n_code_bytes, - e.is_verified, - e.contract_name, - CAST(NULL AS VARCHAR) AS compiler_version - FROM page - LEFT JOIN recent_enrichment e ON page.contract_address = e.contract_address - ORDER BY page.block_number DESC, page.create_index DESC - "# - } else if has_zellic && has_hash_metadata && cursor.is_some() { - r#" - WITH page AS ( - SELECT - z.contract_address, - z.block_number, - z.create_index, - z.bytecode_hash - FROM zellic_contracts z - WHERE z.block_number IS NOT NULL - AND ( - z.block_number < ? - OR (z.block_number = ? AND z.create_index < ?) - ) - ORDER BY z.block_number DESC, z.create_index DESC - LIMIT ? - ) - SELECT - page.contract_address, - page.block_number, - page.create_index, - CAST(NULL AS BLOB) AS deployer, - b.n_code_bytes, - e.is_verified, - e.contract_name, - h.compiler_version - FROM page - LEFT JOIN zellic_bytecodes b ON page.bytecode_hash = b.code_hash - LEFT JOIN recent_enrichment e ON page.contract_address = e.contract_address - LEFT JOIN bytecode_metadata_by_hash h ON page.bytecode_hash = h.code_hash - ORDER BY page.block_number DESC, page.create_index DESC - "# - } else if has_zellic && has_hash_metadata { - r#" - WITH page AS ( - SELECT - z.contract_address, - z.block_number, - z.create_index, - z.bytecode_hash - FROM zellic_contracts z - WHERE z.block_number IS NOT NULL - ORDER BY z.block_number DESC, z.create_index DESC - LIMIT ? - ) - SELECT - page.contract_address, - page.block_number, - page.create_index, - CAST(NULL AS BLOB) AS deployer, - b.n_code_bytes, - e.is_verified, - e.contract_name, - h.compiler_version - FROM page - LEFT JOIN zellic_bytecodes b ON page.bytecode_hash = b.code_hash - LEFT JOIN recent_enrichment e ON page.contract_address = e.contract_address - LEFT JOIN bytecode_metadata_by_hash h ON page.bytecode_hash = h.code_hash - ORDER BY page.block_number DESC, page.create_index DESC - "# - } else if has_zellic && cursor.is_some() { - r#" - WITH page AS ( - SELECT - z.contract_address, - z.block_number, - z.create_index, - z.bytecode_hash - FROM zellic_contracts z - WHERE z.block_number IS NOT NULL - AND ( - z.block_number < ? - OR (z.block_number = ? AND z.create_index < ?) - ) - ORDER BY z.block_number DESC, z.create_index DESC - LIMIT ? - ) - SELECT - page.contract_address, - page.block_number, - page.create_index, - CAST(NULL AS BLOB) AS deployer, - b.n_code_bytes, - e.is_verified, - e.contract_name, - CAST(NULL AS VARCHAR) AS compiler_version - FROM page - LEFT JOIN zellic_bytecodes b ON page.bytecode_hash = b.code_hash - LEFT JOIN recent_enrichment e ON page.contract_address = e.contract_address - ORDER BY page.block_number DESC, page.create_index DESC - "# - } else if has_zellic { - r#" - WITH page AS ( - SELECT - z.contract_address, - z.block_number, - z.create_index, - z.bytecode_hash - FROM zellic_contracts z - WHERE z.block_number IS NOT NULL - ORDER BY z.block_number DESC, z.create_index DESC - LIMIT ? - ) - SELECT - page.contract_address, - page.block_number, - page.create_index, - CAST(NULL AS BLOB) AS deployer, - b.n_code_bytes, - e.is_verified, - e.contract_name, - CAST(NULL AS VARCHAR) AS compiler_version - FROM page - LEFT JOIN zellic_bytecodes b ON page.bytecode_hash = b.code_hash - LEFT JOIN recent_enrichment e ON page.contract_address = e.contract_address - ORDER BY page.block_number DESC, page.create_index DESC - "# - } else if cursor.is_some() { - r#" - WITH page AS ( - SELECT - c.contract_address, - c.block_number, - c.create_index, - c.deployer, - c.n_code_bytes - FROM recent_contracts_all c - WHERE c.block_number IS NOT NULL - AND ( - c.block_number < ? - OR (c.block_number = ? AND c.create_index < ?) - ) - ORDER BY c.block_number DESC, c.create_index DESC - LIMIT ? - ) - SELECT - page.contract_address, - page.block_number, - page.create_index, - page.deployer, - page.n_code_bytes, - e.is_verified, - e.contract_name, - b.compiler_version - FROM page - LEFT JOIN recent_enrichment e ON page.contract_address = e.contract_address - LEFT JOIN bytecode_metadata_current b ON page.contract_address = b.contract_address - ORDER BY page.block_number DESC, page.create_index DESC - "# - } else { - r#" - WITH page AS ( - SELECT - c.contract_address, - c.block_number, - c.create_index, - c.deployer, - c.n_code_bytes - FROM recent_contracts_all c - WHERE c.block_number IS NOT NULL - ORDER BY c.block_number DESC, c.create_index DESC - LIMIT ? - ) - SELECT - page.contract_address, - page.block_number, - page.create_index, - page.deployer, - page.n_code_bytes, - e.is_verified, - e.contract_name, - b.compiler_version - FROM page - LEFT JOIN recent_enrichment e ON page.contract_address = e.contract_address - LEFT JOIN bytecode_metadata_current b ON page.contract_address = b.contract_address - ORDER BY page.block_number DESC, page.create_index DESC - "# - }; - let mut stmt = conn.prepare(sql)?; - let rows = if let Some(cursor) = cursor { - stmt.query_map( - params![ - cursor.block_number as i64, - cursor.block_number as i64, - cursor.create_index as i64, - page_limit - ], - read_recent_row, - )? - } else { - stmt.query_map(params![page_limit], read_recent_row)? - }; - let mut out = Vec::new(); - for r in rows { - let (addr, block, create_index, deployer, n_code, verified, name, compiler) = - r?; - let block_u64 = block as u64; - let is_verified = if registry_loaded { - Some(verified.unwrap_or(false)) - } else { - verified - }; - out.push(RecentContract { - address: format!("0x{}", hex::encode(&addr)), - block_number: block_u64, - create_index: create_index as u64, - timestamp: crate::blocks::block_timestamp(chain_id, block_u64), - deployer: format!("0x{}", hex::encode(deployer.unwrap_or_default())), - n_code_bytes: n_code.unwrap_or(0) as u64, - is_verified, - contract_name: name, - compiler_version: compiler, - }); - } - let last_block = out.last().map(|contract| contract.block_number); - let has_older_external = use_external_recent - && last_block - .map(|block| { - parquet_files - .iter() - .cloned() - .map(contract_file_with_range) - .filter(|file| { - file.chain_id == Some(chain_id) - || (file.chain_id.is_none() - && chain_id == ETHEREUM_CHAIN_ID) - }) - .filter_map(|file| file.end_block) - .any(|end_block| end_block < block) - }) - .unwrap_or(false); - let has_older_zellic = use_external_recent - && last_block - .zip(max_zellic_block) - .map(|(block, zellic_block)| zellic_block < block) - .unwrap_or(false); - Ok((out, has_older_external || has_older_zellic)) - }) - .await - .map_err(|e| anyhow!("join error: {}", e))??; - let has_more = - contracts.len() > limit as usize || (may_have_more_external && !contracts.is_empty()); - if has_more { - contracts.truncate(limit as usize); - } - Ok(RecentPage { - contracts, - has_more, - }) - } - - pub async fn highest_block(&self, chain_id: u64) -> Result> { - let inner = self.inner.clone(); - let data_dir = self.data_dir.clone(); - let contracts_glob = self.contracts_glob.clone(); - tokio::task::spawn_blocking(move || -> Result> { - let parquet_files = list_contract_parquet_files(&data_dir, &contracts_glob)?; - let parquet_filename_max = max_contract_file_block_for_chain(&parquet_files, chain_id); - let conn = inner.blocking_lock(); - let zellic_max = - if chain_id == ETHEREUM_CHAIN_ID && table_exists(&conn, "zellic_contracts")? { - let block: Option = conn - .query_row( - "SELECT MAX(block_number) FROM zellic_contracts", - [], - |row| row.get(0), - ) - .unwrap_or(None); - block.map(u64::from) - } else { - None - }; - let parquet_view_max: Option = conn - .query_row( - "SELECT MAX(block_number) FROM parquet_contracts WHERE chain_id = ?", - params![chain_id], - |row| row.get(0), - ) - .unwrap_or(None); - - let contracts_view_max: Option = conn - .query_row( - "SELECT MAX(block_number) FROM contracts WHERE chain_id = ?", - params![chain_id], - |row| row.get(0), - ) - .unwrap_or(None); - Ok([ - parquet_filename_max, - parquet_view_max.map(u64::from), - contracts_view_max.map(u64::from), - zellic_max, - ] - .into_iter() - .flatten() - .max()) - }) - .await - .map_err(|e| anyhow!("join error: {}", e))? - } - - pub async fn highest_contract_block(&self, chain_id: u64) -> Result> { - let inner = self.inner.clone(); - tokio::task::spawn_blocking(move || -> Result> { - let conn = inner.blocking_lock(); - let zellic_max = if chain_id == ETHEREUM_CHAIN_ID - && table_exists(&conn, "zellic_block_counts")? - { - let block: Option = conn - .query_row( - "SELECT MAX(block_number) FROM zellic_block_counts", - [], - |row| row.get(0), - ) - .unwrap_or(None); - block.map(u64::from) - } else if chain_id == ETHEREUM_CHAIN_ID && table_exists(&conn, "zellic_contracts")? { - let block: Option = conn - .query_row( - "SELECT MAX(block_number) FROM zellic_contracts", - [], - |row| row.get(0), - ) - .unwrap_or(None); - block.map(u64::from) - } else { - None - }; - let parquet_max: Option = conn - .query_row( - "SELECT MAX(block_number) FROM parquet_contracts WHERE chain_id = ?", - params![chain_id], - |row| row.get(0), - ) - .unwrap_or(None); - Ok([zellic_max, parquet_max.map(u64::from)] - .into_iter() - .flatten() - .max()) - }) - .await - .map_err(|e| anyhow!("join error: {}", e))? - } -} - -#[cfg(test)] -mod tests { - use std::{ - fs, - path::{Path, PathBuf}, - time::{SystemTime, UNIX_EPOCH}, - }; - - use duckdb::Connection; - - use crate::chains::ETHEREUM_CHAIN_ID; - - use super::{ - max_contract_file_block_for_chain, recent_contract_parquet_files, Db, RecentCursor, - }; - - struct TestDir { - path: PathBuf, - } - - impl TestDir { - fn new(name: &str) -> Self { - let unique = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_nanos(); - let path = std::env::temp_dir().join(format!( - "blink_db_test_{}_{}_{}", - std::process::id(), - name, - unique - )); - fs::create_dir_all(&path).unwrap(); - Self { path } - } - } - - impl Drop for TestDir { - fn drop(&mut self) { - let _ = fs::remove_dir_all(&self.path); - } - } - - fn make_bytes(value: u8, len: usize) -> Vec { - vec![value; len] - } - - fn insert_zellic_snapshot(data_dir: &Path) { - let conn = Connection::open(data_dir.join("blink.duckdb")).unwrap(); - conn.execute_batch( - r#" - CREATE TABLE zellic_bytecodes ( - code_hash BLOB, - code BLOB, - n_code_bytes UINTEGER - ); - CREATE TABLE zellic_contracts ( - contract_address BLOB, - bytecode_hash BLOB, - block_number UINTEGER, - create_index UINTEGER, - chain_id UBIGINT - ); - "#, - ) - .unwrap(); - conn.execute( - "INSERT INTO zellic_bytecodes VALUES (?, ?, ?)", - duckdb::params![make_bytes(1, 32), make_bytes(0x60, 4), 4u32], - ) - .unwrap(); - conn.execute( - "INSERT INTO zellic_contracts VALUES (?, ?, ?, ?, ?)", - duckdb::params![make_bytes(2, 20), make_bytes(1, 32), 100u32, 0u32, 1u64], - ) - .unwrap(); - } - - fn write_backfill_parquet(data_dir: &Path) { - let ethereum_path = data_dir.join("contracts__0000000200__0000000200.parquet"); - let ethereum_path_sql = ethereum_path.display().to_string().replace('\'', "''"); - let gnosis_path = - data_dir.join("contracts__chain_0000000100__0000000300__0000000300.parquet"); - let gnosis_path_sql = gnosis_path.display().to_string().replace('\'', "''"); - let conn = Connection::open_in_memory().unwrap(); - conn.execute_batch(&format!( - r#" - COPY ( - SELECT - 200::UINTEGER AS block_number, - unhex(repeat('03', 32)) AS block_hash, - 0::UINTEGER AS create_index, - unhex(repeat('04', 32)) AS transaction_hash, - unhex(repeat('05', 20)) AS contract_address, - unhex(repeat('06', 20)) AS deployer, - unhex(repeat('07', 20)) AS factory, - unhex('6000') AS init_code, - unhex('6001') AS code, - unhex(repeat('08', 32)) AS init_code_hash, - 2::UINTEGER AS n_init_code_bytes, - 2::UINTEGER AS n_code_bytes, - unhex(repeat('09', 32)) AS code_hash, - 1::UBIGINT AS chain_id - ) TO '{ethereum_path_sql}' (FORMAT PARQUET); - - COPY ( - SELECT - 300::UINTEGER AS block_number, - unhex(repeat('13', 32)) AS block_hash, - 0::UINTEGER AS create_index, - unhex(repeat('14', 32)) AS transaction_hash, - unhex(repeat('15', 20)) AS contract_address, - unhex(repeat('16', 20)) AS deployer, - unhex(repeat('17', 20)) AS factory, - unhex('6000') AS init_code, - unhex('6001') AS code, - unhex(repeat('18', 32)) AS init_code_hash, - 2::UINTEGER AS n_init_code_bytes, - 2::UINTEGER AS n_code_bytes, - unhex(repeat('19', 32)) AS code_hash, - 100::UBIGINT AS chain_id - ) TO '{gnosis_path_sql}' (FORMAT PARQUET); - "# - )) - .unwrap(); - } - - fn write_sparse_gnosis_parquet(data_dir: &Path) { - let path = data_dir.join("contracts__chain_0000000100__0000000300__0000000500.parquet"); - let path_sql = path.display().to_string().replace('\'', "''"); - let conn = Connection::open_in_memory().unwrap(); - conn.execute_batch(&format!( - r#" - COPY ( - SELECT - 300::UINTEGER AS block_number, - unhex(repeat('13', 32)) AS block_hash, - 0::UINTEGER AS create_index, - unhex(repeat('14', 32)) AS transaction_hash, - unhex(repeat('15', 20)) AS contract_address, - unhex(repeat('16', 20)) AS deployer, - unhex(repeat('17', 20)) AS factory, - unhex('6000') AS init_code, - unhex('6001') AS code, - unhex(repeat('18', 32)) AS init_code_hash, - 2::UINTEGER AS n_init_code_bytes, - 2::UINTEGER AS n_code_bytes, - unhex(repeat('19', 32)) AS code_hash, - 100::UBIGINT AS chain_id - ) TO '{path_sql}' (FORMAT PARQUET); - "# - )) - .unwrap(); - } - - #[test] - fn recent_parquet_selection_uses_filename_block_ranges() { - let files = vec![ - PathBuf::from("/tmp/contracts__0021850001__0021950000.parquet"), - PathBuf::from("/tmp/tail__chain_0000000001__0025330032__0025330040.parquet"), - PathBuf::from("/tmp/tail__chain_0000000001__0025330041__0025330048.parquet"), - PathBuf::from("/tmp/tail__chain_0000000100__0040000001__0040000048.parquet"), - ]; - - assert_eq!( - max_contract_file_block_for_chain(&files, ETHEREUM_CHAIN_ID), - Some(25_330_048) - ); - assert_eq!( - max_contract_file_block_for_chain(&files, 100), - Some(40_000_048) - ); - - let latest = recent_contract_parquet_files(&files, ETHEREUM_CHAIN_ID, None); - assert_eq!( - latest[0].file_name().and_then(|name| name.to_str()), - Some("tail__chain_0000000001__0025330041__0025330048.parquet") - ); - - let gnosis_latest = recent_contract_parquet_files(&files, 100, None); - assert_eq!( - gnosis_latest[0].file_name().and_then(|name| name.to_str()), - Some("tail__chain_0000000100__0040000001__0040000048.parquet") - ); - - let cursor_page = recent_contract_parquet_files( - &files, - ETHEREUM_CHAIN_ID, - Some(RecentCursor { - block_number: 25_330_040, - create_index: 0, - }), - ); - assert_eq!( - cursor_page[0].file_name().and_then(|name| name.to_str()), - Some("tail__chain_0000000001__0025330032__0025330040.parquet") - ); - } - - #[tokio::test] - async fn stats_and_recent_include_parquet_rows_newer_than_zellic() { - let dir = TestDir::new("parquet_newer_than_zellic"); - insert_zellic_snapshot(&dir.path); - write_backfill_parquet(&dir.path); - - let db = Db::open_with_mode(&dir.path, "*.parquet", false).unwrap(); - - let stats = db.stats(ETHEREUM_CHAIN_ID).await.unwrap(); - assert_eq!(stats.total_contracts, 2); - assert_eq!(stats.first_block, 100); - assert_eq!(stats.last_block, 200); - - let deploys = db - .deploys_over_time(ETHEREUM_CHAIN_ID, 100, Some((0, 250))) - .await - .unwrap(); - assert_eq!( - deploys - .iter() - .map(|bucket| (bucket.block_start, bucket.count)) - .collect::>(), - vec![(100, 1), (200, 1)] - ); - - let recent = db - .recent_contracts(ETHEREUM_CHAIN_ID, 5, None) - .await - .unwrap(); - assert_eq!(recent.contracts.len(), 1); - assert_eq!(recent.contracts[0].block_number, 200); - assert_eq!( - recent.contracts[0].address, - format!("0x{}", hex::encode(make_bytes(5, 20))) - ); - } - - #[tokio::test] - async fn chart_anchor_uses_latest_contract_row_not_parquet_filename_end() { - let dir = TestDir::new("chart_anchor_actual_contract_block"); - write_sparse_gnosis_parquet(&dir.path); - - let db = Db::open_with_mode(&dir.path, "*.parquet", false).unwrap(); - - assert_eq!(db.highest_block(100).await.unwrap(), Some(500)); - assert_eq!(db.highest_contract_block(100).await.unwrap(), Some(300)); - } - - #[tokio::test] - async fn stats_and_recent_filter_gnosis_chain() { - let dir = TestDir::new("gnosis_chain_filter"); - insert_zellic_snapshot(&dir.path); - write_backfill_parquet(&dir.path); - - let db = Db::open_with_mode(&dir.path, "*.parquet", false).unwrap(); - - let stats = db.stats(100).await.unwrap(); - assert_eq!(stats.total_contracts, 1); - assert_eq!(stats.first_block, 300); - assert_eq!(stats.last_block, 300); - - let recent = db.recent_contracts(100, 5, None).await.unwrap(); - assert_eq!(recent.contracts.len(), 1); - assert_eq!(recent.contracts[0].block_number, 300); - assert_eq!( - recent.contracts[0].address, - format!("0x{}", hex::encode(make_bytes(0x15, 20))) - ); - } - - #[tokio::test] - async fn recent_does_not_fall_back_to_ethereum_for_other_chains() { - let dir = TestDir::new("no_cross_chain_recent_fallback"); - insert_zellic_snapshot(&dir.path); - - let db = Db::open_with_mode(&dir.path, "*.parquet", false).unwrap(); - - let recent = db.recent_contracts(100, 5, None).await.unwrap(); - assert!(recent.contracts.is_empty()); - assert!(!recent.has_more); - } - - #[tokio::test] - async fn dashboard_sql_scopes_contract_metadata_by_chain() { - let dir = TestDir::new("query_chain_scope"); - insert_zellic_snapshot(&dir.path); - write_backfill_parquet(&dir.path); - - let db = Db::open_with_mode(&dir.path, "*.parquet", false).unwrap(); - - let result = db - .query_sql( - "SELECT chain_id, block_number FROM contract_metadata ORDER BY block_number" - .to_string(), - 10, - Some(100), - ) - .await - .unwrap(); - - assert_eq!(result.row_count, 1); - assert_eq!(result.rows[0][0], serde_json::json!(100)); - assert_eq!(result.rows[0][1], serde_json::json!(300)); - } -} diff --git a/src/db/explorer.rs b/src/db/explorer.rs new file mode 100644 index 0000000..361061f --- /dev/null +++ b/src/db/explorer.rs @@ -0,0 +1,375 @@ +//! Materialized backing table for the SQL explorer (`POST /api/query`). + +use anyhow::{Context, Result}; +use duckdb::{params, Connection}; + +use super::{table_exists, views, Db}; + +const EXPLORER_FINGERPRINT_KEY: &str = "explorer_fingerprint"; +/// Rows per build slice — bounds the join/sort working set per transaction. +const EXPLORER_SLICE_TARGET_ROWS: u64 = 2_000_000; + +pub(crate) fn ensure_explorer_schema(conn: &Connection) -> Result<()> { + conn.execute_batch( + r#" + CREATE TABLE IF NOT EXISTS blink_state ( + key VARCHAR PRIMARY KEY, + value VARCHAR NOT NULL + ); + CREATE TABLE IF NOT EXISTS contract_metadata_bounds ( + chain_id UBIGINT PRIMARY KEY, + max_block UBIGINT NOT NULL + ); + "#, + ) + .context("create explorer state schema") +} + +/// Inputs that change what the materialized rows would contain (deployment +/// growth is handled separately: tail growth beyond the bounds is served +/// live, backfills below the bounds are detected by `counts_below_bounds`). +fn input_fingerprint(conn: &Connection) -> Result { + let (meta_rows, meta_latest): (i64, Option) = + if table_exists(conn, "bytecode_metadata_by_hash")? { + conn.query_row( + "SELECT COUNT(*), CAST(MAX(decoded_at) AS VARCHAR) FROM bytecode_metadata_by_hash", + [], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .unwrap_or((0, None)) + } else { + (0, None) + }; + let (enrichment_rows, enrichment_latest): (i64, Option) = + if table_exists(conn, "enrichment")? { + conn.query_row( + "SELECT COUNT(*), CAST(MAX(checked_at) AS VARCHAR) FROM enrichment", + [], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .unwrap_or((0, None)) + } else { + (0, None) + }; + let registry_rows: i64 = if table_exists(conn, "verification_registry_imports")? { + conn.query_row( + "SELECT COUNT(*) FROM verification_registry_imports", + [], + |row| row.get(0), + ) + .unwrap_or(0) + } else { + 0 + }; + // Bump the version prefix when the table schema changes so upgraded + // servers rebuild instead of querying a stale shape. + Ok(format!( + "v2|meta:{meta_rows}@{}|enr:{enrichment_rows}@{}|reg:{registry_rows}", + meta_latest.unwrap_or_default(), + enrichment_latest.unwrap_or_default() + )) +} + +fn stored_fingerprint(conn: &Connection) -> Result> { + if !table_exists(conn, "blink_state")? { + return Ok(None); + } + let value: Option = conn + .query_row( + "SELECT value FROM blink_state WHERE key = ?", + params![EXPLORER_FINGERPRINT_KEY], + |row| row.get(0), + ) + .ok(); + Ok(value) +} + +/// Detect deployments added *below* the materialized head (a backfill file +/// landing mid-history): per chain, the deployment count up to the bound must +/// match what was materialized. +fn deployments_backfilled_below_bounds(conn: &Connection) -> Result { + if !table_exists(conn, "contract_metadata_native")? { + return Ok(true); + } + let mismatch: i64 = conn + .query_row( + r#" + SELECT COUNT(*) + FROM ( + SELECT + b.chain_id, + ( + SELECT COUNT(*) FROM contract_deployments_native c + WHERE c.chain_id = b.chain_id AND c.block_number <= b.max_block + ) AS deployed, + ( + SELECT COUNT(*) FROM contract_metadata_native m + WHERE m.chain_id = b.chain_id + ) AS materialized + FROM contract_metadata_bounds b + ) + WHERE deployed != materialized + "#, + [], + |row| row.get(0), + ) + .unwrap_or(1); + if mismatch > 0 { + return Ok(true); + } + // A chain that gained its first deployments after the last build has no + // bounds row at all. + let unbounded_chains: i64 = conn + .query_row( + r#" + SELECT COUNT(*) FROM ( + SELECT DISTINCT chain_id FROM rollup_block_counts + WHERE chain_id NOT IN (SELECT chain_id FROM contract_metadata_bounds) + ) + "#, + [], + |row| row.get(0), + ) + .unwrap_or(1); + Ok(unbounded_chains > 0) +} + +struct ChainSlice { + chain_id: u64, + start_block: u64, + end_block: u64, +} + +fn build_slices(conn: &Connection) -> Result> { + let mut stmt = conn.prepare( + r#" + SELECT chain_id, MIN(block_number), MAX(block_number), SUM(contract_count) + FROM rollup_block_counts + GROUP BY chain_id + ORDER BY chain_id + "#, + )?; + let chains = stmt + .query_map([], |row| { + Ok(( + row.get::<_, u64>(0)?, + row.get::<_, u32>(1)?, + row.get::<_, u32>(2)?, + row.get::<_, i64>(3)?, + )) + })? + .collect::, _>>()?; + + let mut slices = Vec::new(); + for (chain_id, min_block, max_block, rows) in chains { + let (min_block, max_block) = (u64::from(min_block), u64::from(max_block)); + let span = max_block - min_block + 1; + let slice_count = (rows.max(0) as u64) + .div_ceil(EXPLORER_SLICE_TARGET_ROWS) + .max(1); + let slice_blocks = span.div_ceil(slice_count).max(1); + let mut start = min_block; + while start <= max_block { + let end = (start + slice_blocks - 1).min(max_block); + slices.push(ChainSlice { + chain_id, + start_block: start, + end_block: end, + }); + start = end + 1; + } + } + Ok(slices) +} + +impl Db { + /// Bring the materialized explorer table up to date. Returns whether a + /// rebuild ran. Takes and releases the writer lock per slice so the tail + /// loop keeps running during the minutes-long build. + pub(crate) fn refresh_explorer_blocking(&self) -> Result { + if self.read_only { + return Ok(false); + } + + let fingerprint = { + let conn = self.writer.blocking_lock(); + ensure_explorer_schema(&conn)?; + let fingerprint = input_fingerprint(&conn)?; + let unchanged = stored_fingerprint(&conn)?.as_deref() == Some(fingerprint.as_str()) + && !deployments_backfilled_below_bounds(&conn)?; + if unchanged { + return Ok(false); + } + tracing::info!("rebuilding sql explorer table (contract_metadata_native)"); + + // One deduplicated copy of the decode metadata for the whole + // build, so the per-slice joins don't re-run the window function. + // Only the columns the join needs — no hex strings — to keep the + // per-slice hash-build side small on 4GB hosts. + let has_meta = table_exists(&conn, "bytecode_metadata_by_hash")?; + let meta_source = if has_meta { + r#" + SELECT + code_hash, language, compiler_version, has_source_hash, + is_erc20, is_erc721, is_erc1155, is_proxy_eip1967, + is_proxy_minimal, uses_push0, decoded_at + FROM decoded_bytecodes + "# + } else { + r#" + SELECT + CAST(NULL AS BLOB) AS code_hash, + CAST(NULL AS VARCHAR) AS language, + CAST(NULL AS VARCHAR) AS compiler_version, + CAST(false AS BOOLEAN) AS has_source_hash, + CAST(false AS BOOLEAN) AS is_erc20, + CAST(false AS BOOLEAN) AS is_erc721, + CAST(false AS BOOLEAN) AS is_erc1155, + CAST(false AS BOOLEAN) AS is_proxy_eip1967, + CAST(false AS BOOLEAN) AS is_proxy_minimal, + CAST(false AS BOOLEAN) AS uses_push0, + CAST(NULL AS TIMESTAMP) AS decoded_at + WHERE FALSE + "# + }; + conn.execute_batch(&format!( + r#" + CREATE OR REPLACE TEMP TABLE explorer_meta_build AS {meta_source}; + DROP TABLE IF EXISTS contract_metadata_native_build; + CREATE TABLE contract_metadata_native_build ( + chain_id UBIGINT NOT NULL, + block_number UINTEGER NOT NULL, + create_index UINTEGER NOT NULL, + contract_address BLOB NOT NULL, + deployer BLOB, + code_hash BLOB, + n_code_bytes UINTEGER, + language VARCHAR, + compiler_version VARCHAR, + has_source_hash BOOLEAN, + is_erc20 BOOLEAN, + is_erc721 BOOLEAN, + is_erc1155 BOOLEAN, + is_proxy_eip1967 BOOLEAN, + is_proxy_minimal BOOLEAN, + uses_push0 BOOLEAN, + decoded_at TIMESTAMP, + is_verified BOOLEAN, + contract_name VARCHAR, + verification_source VARCHAR, + match_type VARCHAR, + verification_checked_at TIMESTAMP, + is_decoded BOOLEAN + ); + "# + )) + .context("prepare explorer build")?; + fingerprint + }; + + let slices = { + let conn = self.writer.blocking_lock(); + build_slices(&conn)? + }; + let total_slices = slices.len(); + let is_verified_expr = { + let conn = self.writer.blocking_lock(); + if table_exists(&conn, "verification_registry_imports")? { + "COALESCE(e.is_verified, false)" + } else { + "e.is_verified" + } + }; + + for (index, slice) in slices.iter().enumerate() { + let conn = self.writer.blocking_lock(); + let ChainSlice { + chain_id, + start_block, + end_block, + } = slice; + conn.execute_batch(&format!( + r#" + INSERT INTO contract_metadata_native_build + SELECT + c.chain_id, c.block_number, c.create_index, c.contract_address, + c.deployer, c.code_hash, c.n_code_bytes, + m.language, m.compiler_version, + COALESCE(m.has_source_hash, false), + COALESCE(m.is_erc20, false), + COALESCE(m.is_erc721, false), + COALESCE(m.is_erc1155, false), + COALESCE(m.is_proxy_eip1967, false), + COALESCE(m.is_proxy_minimal, false), + COALESCE(m.uses_push0, false), + m.decoded_at, + {is_verified_expr}, + e.contract_name, + e.verification_source, + e.match_type, + e.checked_at, + m.code_hash IS NOT NULL + FROM contract_deployments_native c + LEFT JOIN explorer_meta_build m ON c.code_hash = m.code_hash + -- The block-range condition relies on + -- backfill_enrichment_blocks having run at open: every + -- enrichment row whose address exists in the deployments + -- table carries that deployment's block_number, so the join + -- hash-builds only this slice's enrichment rows instead of + -- the whole table. + LEFT JOIN enrichment_current e + ON c.contract_address = e.contract_address + AND c.chain_id = e.chain_id + AND e.block_number BETWEEN {start_block} AND {end_block} + WHERE c.chain_id = {chain_id} + AND c.block_number BETWEEN {start_block} AND {end_block} + ORDER BY c.block_number, c.create_index; + "# + )) + .with_context(|| { + format!("explorer build slice chain={chain_id} blocks {start_block}-{end_block}") + })?; + if total_slices > 1 { + tracing::info!( + "sql explorer build progress: slice {}/{}", + index + 1, + total_slices + ); + } + } + + { + let conn = self.writer.blocking_lock(); + let fingerprint_sql = fingerprint.replace('\'', "''"); + conn.execute_batch(&format!( + r#" + BEGIN; + DROP TABLE IF EXISTS contract_metadata_native; + ALTER TABLE contract_metadata_native_build RENAME TO contract_metadata_native; + DELETE FROM contract_metadata_bounds; + INSERT INTO contract_metadata_bounds + SELECT chain_id, MAX(block_number)::UBIGINT + FROM contract_metadata_native + GROUP BY chain_id; + INSERT INTO blink_state VALUES ('{EXPLORER_FINGERPRINT_KEY}', '{fingerprint_sql}') + ON CONFLICT (key) DO UPDATE SET value = excluded.value; + COMMIT; + DROP TABLE IF EXISTS explorer_meta_build; + "# + )) + .context("swap explorer table")?; + } + + // The contract_metadata view definition depends on the table's + // existence — rebuild it on every pooled connection. + { + let conn = self.writer.blocking_lock(); + views::create_contract_metadata_view(&conn)?; + } + for reader in self.readers.iter() { + let conn = reader.blocking_lock(); + views::create_contract_metadata_view(&conn)?; + } + Ok(true) + } +} diff --git a/src/db/mod.rs b/src/db/mod.rs new file mode 100644 index 0000000..4a8778e --- /dev/null +++ b/src/db/mod.rs @@ -0,0 +1,298 @@ +//! DuckDB-backed query layer for the dashboard. +//! +//! Architecture, in the order requests hit it: +//! - **Ingest** (`rollups`): every contract parquet file is immutable once +//! written, so it is rolled up exactly once into native DuckDB tables — +//! `contract_deployments_native` (one deduplicated, blob-free row per +//! deployment) plus `rollup_block_counts` / `rollup_code_counts` summaries +//! derived from the deduped delta. Overlapping files therefore cannot +//! double-count. +//! - **Queries** (`queries`): every dashboard endpoint reads only those +//! native tables; no request ever scans parquet. `POST /api/query` keeps a +//! parquet-backed `contracts` view (`views`) for raw SQL access to the full +//! bytecode columns. +//! - **Concurrency**: one writer connection owns ingest; a small pool of +//! reader connections (clones of the same DuckDB instance, so they share +//! the buffer cache) serves queries in parallel instead of serializing on a +//! single mutex. + +use std::{ + path::{Path, PathBuf}, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, +}; + +use anyhow::{anyhow, Context, Result}; +use duckdb::{params, AccessMode, Config, Connection}; +use tokio::sync::Mutex; + +mod explorer; +mod queries; +mod rollups; +mod sql; +mod views; + +pub use queries::{ + CompilerCount, DeployBucket, LanguageCount, RecentContract, RecentCursor, RecentPage, SizeBin, + StandardsBreakdown, Stats, VerifiedRatioBucket, +}; +pub use rollups::invalidate_zellic_rollups; +pub use sql::SqlQueryResult; + +/// Tuning knobs for [`Db::open`]. `Default` keeps DuckDB's own defaults and +/// sizes the reader pool from the host's core count. +#[derive(Debug, Clone, Default)] +pub struct DbOptions { + pub read_only: bool, + /// DuckDB instance-wide memory limit, e.g. `"2GB"`. Worth setting on + /// small hosts; DuckDB otherwise assumes 80% of physical RAM. + pub memory_limit: Option, + /// DuckDB instance-wide thread count. + pub threads: Option, + /// Reader connections for query traffic. `0` = auto. + pub readers: usize, +} + +#[derive(Clone)] +pub struct Db { + writer: Arc>, + readers: Arc>>>, + next_reader: Arc, + data_dir: PathBuf, + contracts_glob: String, + read_only: bool, +} + +impl Db { + pub fn open_with_mode(data_dir: &Path, contracts_glob: &str, read_only: bool) -> Result { + Self::open( + data_dir, + contracts_glob, + DbOptions { + read_only, + ..DbOptions::default() + }, + ) + } + + pub fn open(data_dir: &Path, contracts_glob: &str, options: DbOptions) -> Result { + if !options.read_only { + std::fs::create_dir_all(data_dir) + .with_context(|| format!("create data dir {}", data_dir.display()))?; + } + let db_path = data_dir.join("blink.duckdb"); + + let writer = if options.read_only { + // Read-only connection coexists with an active writer since DuckDB + // only takes an exclusive lock on writers. + let config = Config::default() + .access_mode(AccessMode::ReadOnly) + .context("set read-only access mode")?; + Connection::open_with_flags(&db_path, config) + .with_context(|| format!("open duckdb (read-only) {}", db_path.display()))? + } else { + Connection::open(&db_path) + .with_context(|| format!("open duckdb {}", db_path.display()))? + }; + configure_connection(&writer, data_dir, &options)?; + + if options.read_only { + // Schema and rollups are owned by the writer process. If they are + // not there yet, queries fail gracefully per-request. + if !rollups::rollups_ready(&writer)? { + tracing::warn!( + "read-only mode: rollup tables missing — run a writable `blink serve` once \ + to build them; dashboard queries will fail until then" + ); + } + } else { + views::ensure_schema(&writer)?; + rollups::ensure_rollup_schema(&writer)?; + rollups::sync_rollups(&writer, data_dir, contracts_glob)?; + rollups::backfill_enrichment_blocks(&writer)?; + } + + let files = rollups::list_contract_parquet_files(data_dir, contracts_glob)?; + views::rebuild_query_views(&writer, &files)?; + + let reader_count = if options.readers == 0 { + default_reader_count() + } else { + options.readers + }; + let mut readers = Vec::with_capacity(reader_count); + for _ in 0..reader_count { + let conn = writer + .try_clone() + .context("clone duckdb reader connection")?; + // Temp views live in a per-connection schema, so each reader + // needs its own copy. + views::rebuild_query_views(&conn, &files)?; + readers.push(Arc::new(Mutex::new(conn))); + } + + Ok(Self { + writer: Arc::new(Mutex::new(writer)), + readers: Arc::new(readers), + next_reader: Arc::new(AtomicUsize::new(0)), + data_dir: data_dir.to_path_buf(), + contracts_glob: contracts_glob.to_string(), + read_only: options.read_only, + }) + } + + /// Pick up newly written parquet files: ingest them into the rollups and + /// point the raw-SQL `contracts` view at the new file list. Called by the + /// background tail loop after each extraction. + pub async fn refresh(&self) -> Result<()> { + let this = self.clone(); + tokio::task::spawn_blocking(move || this.refresh_blocking()) + .await + .map_err(|e| anyhow!("join error: {}", e))? + } + + fn refresh_blocking(&self) -> Result<()> { + if !self.read_only { + let writer = self.writer.blocking_lock(); + rollups::sync_rollups(&writer, &self.data_dir, &self.contracts_glob)?; + } + let files = rollups::list_contract_parquet_files(&self.data_dir, &self.contracts_glob)?; + { + let writer = self.writer.blocking_lock(); + views::rebuild_parquet_views(&writer, &files)?; + } + for reader in self.readers.iter() { + let conn = reader.blocking_lock(); + views::rebuild_parquet_views(&conn, &files)?; + } + Ok(()) + } + + /// Bring the materialized SQL-explorer table (`contract_metadata_native`) + /// up to date if its inputs changed. Safe to run while serving: queries + /// use the previous copy (or the live-join fallback) until the swap, and + /// the writer lock is released between build slices. Returns whether a + /// rebuild ran. + pub async fn refresh_explorer(&self) -> Result { + let this = self.clone(); + tokio::task::spawn_blocking(move || this.refresh_explorer_blocking()) + .await + .map_err(|e| anyhow!("join error: {}", e))? + } + + /// Run raw SQL on the writer connection. Intended for tests and admin + /// tooling, not request paths. + pub async fn execute_batch(&self, sql: String) -> Result<()> { + let writer = self.writer.clone(); + tokio::task::spawn_blocking(move || -> Result<()> { + writer + .blocking_lock() + .execute_batch(&sql) + .context("execute batch") + }) + .await + .map_err(|e| anyhow!("join error: {}", e))? + } + + fn reader(&self) -> Arc> { + if self.readers.is_empty() { + return self.writer.clone(); + } + let idx = self.next_reader.fetch_add(1, Ordering::Relaxed) % self.readers.len(); + self.readers[idx].clone() + } + + /// Run a closure against a pooled reader connection on the blocking + /// thread pool. All dashboard queries go through this. + async fn run_read(&self, f: F) -> Result + where + F: FnOnce(&Connection) -> Result + Send + 'static, + T: Send + 'static, + { + let conn = self.reader(); + tokio::task::spawn_blocking(move || { + let conn = conn.blocking_lock(); + f(&conn) + }) + .await + .map_err(|e| anyhow!("join error: {}", e))? + } + + pub(crate) fn data_dir(&self) -> &Path { + &self.data_dir + } + + pub(crate) fn contracts_glob(&self) -> &str { + &self.contracts_glob + } +} + +fn default_reader_count() -> usize { + std::thread::available_parallelism() + .map(|n| n.get() / 2) + .unwrap_or(2) + .clamp(2, 4) +} + +fn configure_connection(conn: &Connection, data_dir: &Path, options: &DbOptions) -> Result<()> { + conn.execute_batch("SET preserve_insertion_order=false;") + .context("configure DuckDB insertion-order preservation")?; + if let Some(memory_limit) = options.memory_limit.as_deref() { + conn.execute_batch(&format!( + "SET memory_limit='{}';", + memory_limit.replace('\'', "''") + )) + .context("configure DuckDB memory limit")?; + } + if let Some(threads) = options.threads { + conn.execute_batch(&format!("SET threads={};", threads.max(1))) + .context("configure DuckDB threads")?; + } + let temp_dir = data_dir.join(".blink").join("duckdb-tmp"); + match std::fs::create_dir_all(&temp_dir) { + Ok(()) => conn + .execute_batch(&format!( + "SET temp_directory='{}';", + sql_string_literal(&temp_dir) + )) + .context("configure DuckDB temp directory")?, + Err(err) => tracing::warn!( + "could not create DuckDB temp dir {}: {}; using DuckDB default temp directory", + temp_dir.display(), + err + ), + } + Ok(()) +} + +fn sql_string_literal(value: &Path) -> String { + value.display().to_string().replace('\'', "''") +} + +pub(crate) fn table_exists(conn: &Connection, table: &str) -> Result { + let count: i64 = conn + .query_row( + "SELECT COUNT(*) FROM information_schema.tables WHERE table_name = ?", + params![table], + |row| row.get(0), + ) + .unwrap_or(0); + Ok(count > 0) +} + +pub(crate) fn column_exists(conn: &Connection, table: &str, column: &str) -> Result { + let count: i64 = conn + .query_row( + r#" + SELECT COUNT(*) + FROM information_schema.columns + WHERE table_name = ? AND column_name = ? + "#, + params![table, column], + |row| row.get(0), + ) + .unwrap_or(0); + Ok(count > 0) +} diff --git a/src/db/queries.rs b/src/db/queries.rs new file mode 100644 index 0000000..ce2b00b --- /dev/null +++ b/src/db/queries.rs @@ -0,0 +1,1044 @@ +use std::{ + collections::{HashMap, HashSet}, + time::Instant, +}; + +use anyhow::{Context, Result}; +use chrono::{DateTime, Utc}; +use duckdb::{params, Connection, Row}; + +use super::{column_exists, rollups, sql, table_exists, Db}; + +/// Fast path for the "recent deployments" page: scan only this many blocks +/// below the cursor/head first, and fall back to an unbounded scan when the +/// window doesn't fill the page (tiny datasets, sparse chains). +const RECENT_SCAN_WINDOW_BLOCKS: u64 = 100_000; + +#[derive(Debug, Clone, serde::Serialize, utoipa::ToSchema)] +pub struct Stats { + pub total_contracts: u64, + pub verified_count: u64, + pub unverified_count: u64, + pub verified_pct: f64, + pub last_block: u64, + pub first_block: u64, + pub enrichment_coverage_pct: f64, + pub last_updated: DateTime, +} + +#[derive(Debug, Clone, serde::Serialize, utoipa::ToSchema)] +pub struct DeployBucket { + pub block_start: u64, + pub block_end: u64, + pub timestamp: DateTime, + pub count: u64, +} + +#[derive(Debug, Clone, serde::Serialize, utoipa::ToSchema)] +pub struct VerifiedRatioBucket { + pub block_start: u64, + pub block_end: u64, + pub timestamp: DateTime, + pub verified: u64, + pub unverified: u64, + pub unknown: u64, +} + +#[derive(Debug, Clone, serde::Serialize, utoipa::ToSchema)] +pub struct SizeBin { + pub label: String, + pub size_min: u64, + pub size_max: u64, + pub count: u64, +} + +#[derive(Debug, Clone, serde::Serialize, utoipa::ToSchema)] +pub struct CompilerCount { + pub compiler_version: String, + pub count: u64, +} + +#[derive(Debug, Clone, serde::Serialize, utoipa::ToSchema)] +pub struct LanguageCount { + pub language: String, + pub count: u64, +} + +#[derive(Debug, Clone, Default, serde::Serialize, utoipa::ToSchema)] +pub struct StandardsBreakdown { + pub erc20: u64, + pub erc721: u64, + pub erc1155: u64, + pub proxy_eip1967: u64, + pub proxy_minimal: u64, + pub uses_push0: u64, + pub has_source_hash: u64, + pub total_decoded: u64, +} + +#[derive(Debug, Clone, serde::Serialize, utoipa::ToSchema)] +pub struct RecentContract { + pub address: String, + pub block_number: u64, + pub create_index: u64, + pub timestamp: DateTime, + pub deployer: String, + pub n_code_bytes: u64, + pub is_verified: Option, + pub contract_name: Option, + pub compiler_version: Option, +} + +#[derive(Debug, Clone, Copy)] +pub struct RecentCursor { + pub block_number: u64, + pub create_index: u64, +} + +#[derive(Debug, Clone, serde::Serialize, utoipa::ToSchema)] +pub struct RecentPage { + pub contracts: Vec, + pub has_more: bool, +} + +/// One row of the recent-deployments page before decoration. +struct RecentPageRow { + address: Vec, + block_number: u32, + create_index: u32, + deployer: Option>, + n_code_bytes: Option, + code_hash: Option>, +} + +fn read_u64_pair(row: &Row<'_>) -> duckdb::Result<(u64, u64)> { + Ok((row.get::<_, u64>(0)?, row.get::<_, u64>(1)?)) +} + +fn read_verified_bucket_row(row: &Row<'_>) -> duckdb::Result<(u64, u64, u64, u64)> { + Ok(( + row.get::<_, u64>(0)?, + row.get::<_, u64>(1)?, + row.get::<_, u64>(2)?, + row.get::<_, u64>(3)?, + )) +} + +fn read_string_u64_pair(row: &Row<'_>) -> duckdb::Result<(String, u64)> { + Ok((row.get::<_, String>(0)?, row.get::<_, u64>(1)?)) +} + +fn verification_registry_loaded(conn: &Connection, chain_id: u64) -> Result { + if !table_exists(conn, "verification_registry_imports")? { + return Ok(false); + } + let count: i64 = conn + .query_row( + "SELECT COUNT(*) FROM verification_registry_imports WHERE source = 'verifier_alliance' AND chain_id = ?", + params![chain_id], + |row| row.get(0), + ) + .unwrap_or(0); + Ok(count > 0) +} + +fn range_filter(block_range: Option<(u64, u64)>) -> String { + block_range + .map(|(start, end)| format!("AND block_number BETWEEN {start} AND {end}")) + .unwrap_or_default() +} + +/// Per-code-hash deployment counts feeding the size/compiler/standards +/// breakdowns. No range → the full-history rollup. Ranged → fully covered +/// block buckets come from `rollup_code_block_counts` and only the partial +/// edge buckets scan the deployments table, so even year-wide custom ranges +/// stay rollup-speed. Consumers SUM over the rows, so a code_hash may appear +/// once per part. +fn code_counts_source(chain_id: u64, block_range: Option<(u64, u64)>) -> String { + let Some((start, end)) = block_range else { + return format!( + r#" + SELECT code_hash, n_code_bytes, contract_count + FROM rollup_code_counts + WHERE chain_id = {chain_id} + "# + ); + }; + + let bucket = rollups::CODE_ROLLUP_BUCKET_BLOCKS; + // Bucket b covers [b*bucket, (b+1)*bucket - 1]; it is fully inside the + // range iff b >= ceil(start/bucket) and b <= (end+1)/bucket - 1. + let first_full = start.div_ceil(bucket); + let full_bucket_end = (end + 1) / bucket; // exclusive + if first_full >= full_bucket_end { + return deployments_code_scan(chain_id, start, end); + } + let interior_hi = full_bucket_end - 1; + let mut parts = vec![format!( + r#" + SELECT code_hash, n_code_bytes, contract_count + FROM rollup_code_block_counts + WHERE chain_id = {chain_id} + AND block_bucket BETWEEN {first_full} AND {interior_hi} + "# + )]; + let interior_start_block = first_full * bucket; + if start < interior_start_block { + parts.push(deployments_code_scan( + chain_id, + start, + interior_start_block - 1, + )); + } + let trailing_start_block = (interior_hi + 1) * bucket; + if end >= trailing_start_block { + parts.push(deployments_code_scan(chain_id, trailing_start_block, end)); + } + parts.join("\nUNION ALL\n") +} + +/// Whether the materialized explorer table can back the metadata aggregates +/// (compilers/standards/sizes/languages). It denormalizes decode metadata per +/// deployment, sorted by (chain, block), so ranged aggregates are pruned +/// scans with no join against the multi-million-row metadata table. +/// +/// The column check matters: while a schema-upgrading rebuild runs (or if one +/// failed), the previous-generation table is still what's on disk — fall back +/// to the join path rather than referencing columns it doesn't have yet. +fn explorer_backed(conn: &Connection) -> Result { + Ok(table_exists(conn, "contract_metadata_native")? + && table_exists(conn, "contract_metadata_bounds")? + && column_exists(conn, "contract_metadata_native", "is_decoded")?) +} + +/// Per-deployment metadata rows for aggregate endpoints: the materialized +/// table for everything up to its bounds, plus live (undecorated) rows for +/// newer blocks — fresh contracts have rarely been decoded yet, matching the +/// join-based semantics. +fn explorer_code_rows_source(chain_id: u64, block_range: Option<(u64, u64)>) -> String { + let filter = range_filter(block_range); + format!( + r#" + SELECT + n_code_bytes, language, compiler_version, has_source_hash, + is_erc20, is_erc721, is_erc1155, is_proxy_eip1967, + is_proxy_minimal, uses_push0, is_decoded + FROM contract_metadata_native + WHERE chain_id = {chain_id} + {filter} + UNION ALL + SELECT + c.n_code_bytes, + CAST(NULL AS VARCHAR), CAST(NULL AS VARCHAR), false, + false, false, false, false, + false, false, false + FROM contract_deployments_native c + LEFT JOIN contract_metadata_bounds b ON c.chain_id = b.chain_id + WHERE c.chain_id = {chain_id} + AND (b.chain_id IS NULL OR c.block_number > b.max_block) + {live_filter} + "#, + live_filter = block_range + .map(|(start, end)| format!("AND c.block_number BETWEEN {start} AND {end}")) + .unwrap_or_default() + ) +} + +fn deployments_code_scan(chain_id: u64, start: u64, end: u64) -> String { + format!( + r#" + SELECT + code_hash, + any_value(n_code_bytes)::UINTEGER AS n_code_bytes, + COUNT(*)::UBIGINT AS contract_count + FROM contract_deployments_native + WHERE chain_id = {chain_id} + AND code_hash IS NOT NULL + AND block_number BETWEEN {start} AND {end} + GROUP BY code_hash + "# + ) +} + +fn max_indexed_block(conn: &Connection, chain_id: u64) -> Result> { + let block: Option = conn + .query_row( + "SELECT MAX(block_number) FROM rollup_block_counts WHERE chain_id = ?", + params![chain_id], + |row| row.get(0), + ) + .unwrap_or(None); + Ok(block.map(u64::from)) +} + +impl Db { + pub async fn query_sql( + &self, + sql: String, + limit: u32, + chain_id: Option, + ) -> Result { + let normalized = sql::normalize_read_only_sql(&sql)?; + let limit = limit.clamp(1, 1_000); + self.run_read(move |conn| { + let started = Instant::now(); + let wrapped = sql::wrap_dashboard_query(&normalized, limit, chain_id); + let mut stmt = conn.prepare(&wrapped).context("prepare dashboard query")?; + let mut rows = stmt.query([]).context("execute dashboard query")?; + let (columns, column_count) = { + let stmt = rows + .as_ref() + .context("dashboard query statement metadata unavailable")?; + (stmt.column_names(), stmt.column_count()) + }; + let mut out = Vec::new(); + while let Some(row) = rows.next().context("read dashboard query row")? { + let mut values = Vec::with_capacity(column_count); + for idx in 0..column_count { + values.push(sql::value_ref_to_json(row.get_ref(idx)?)); + } + out.push(values); + } + let elapsed_ms = started.elapsed().as_millis(); + if elapsed_ms >= 1_000 { + // The HTTP-level slow log can't see the request body; name + // the offending SQL here so slow explorer queries are + // diagnosable from the journal. + tracing::warn!( + "slow sql explorer query ({}ms): {}", + elapsed_ms, + normalized.chars().take(300).collect::() + ); + } + Ok(super::SqlQueryResult { + columns, + row_count: out.len(), + rows: out, + limit, + elapsed_ms, + }) + }) + .await + } + + pub async fn stats(&self, chain_id: u64) -> Result { + self.run_read(move |conn| { + let (total, first_block, last_block): (i64, Option, Option) = conn + .query_row( + r#" + SELECT + COALESCE(SUM(contract_count), 0)::BIGINT, + MIN(block_number), + MAX(block_number) + FROM rollup_block_counts + WHERE chain_id = ? + "#, + params![chain_id], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .unwrap_or((0, None, None)); + let total = total.max(0) as u64; + + let registry_loaded = verification_registry_loaded(conn, chain_id)?; + let (enriched, verified): (i64, i64) = if registry_loaded { + let verified: i64 = conn + .query_row( + r#" + SELECT COUNT(*) + FROM contract_deployments_native c + JOIN enrichment e + ON c.contract_address = e.contract_address + AND c.chain_id = e.chain_id + WHERE e.is_verified + AND c.chain_id = ? + "#, + params![chain_id], + |row| row.get(0), + ) + .unwrap_or(0); + (total as i64, verified) + } else { + conn.query_row( + "SELECT COUNT(*), COUNT(*) FILTER (WHERE is_verified) FROM enrichment WHERE chain_id = ?", + params![chain_id], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .unwrap_or((0, 0)) + }; + + let verified_count = verified.max(0) as u64; + let enriched_count = enriched.max(0) as u64; + let unverified_count = enriched_count.saturating_sub(verified_count); + let verified_pct = if enriched_count == 0 { + 0.0 + } else { + 100.0 * verified_count as f64 / enriched_count as f64 + }; + let enrichment_coverage_pct = if total == 0 { + 0.0 + } else { + 100.0 * enriched_count as f64 / total as f64 + }; + + Ok(Stats { + total_contracts: total, + verified_count, + unverified_count, + verified_pct, + last_block: last_block.map(u64::from).unwrap_or(0), + first_block: first_block.map(u64::from).unwrap_or(0), + enrichment_coverage_pct, + last_updated: Utc::now(), + }) + }) + .await + } + + pub async fn deploys_over_time( + &self, + chain_id: u64, + bucket_blocks: u64, + block_range: Option<(u64, u64)>, + ) -> Result> { + let bucket_blocks = bucket_blocks.max(1); + self.run_read(move |conn| { + let filter = range_filter(block_range); + let sql = format!( + r#" + SELECT + (block_number // {bucket_blocks})::UBIGINT AS bucket_id, + SUM(contract_count)::UBIGINT AS cnt + FROM rollup_block_counts + WHERE chain_id = {chain_id} + {filter} + GROUP BY bucket_id + ORDER BY bucket_id + "# + ); + let mut stmt = conn.prepare(&sql)?; + let rows = stmt.query_map([], read_u64_pair)?; + let mut out = Vec::new(); + for r in rows { + let (bucket_id, count) = r?; + let block_start = bucket_id * bucket_blocks; + let block_end = block_start + bucket_blocks - 1; + let mid = block_start + bucket_blocks / 2; + out.push(DeployBucket { + block_start, + block_end, + timestamp: crate::blocks::block_timestamp(chain_id, mid), + count, + }); + } + Ok(out) + }) + .await + } + + pub async fn verified_ratio_over_time( + &self, + chain_id: u64, + bucket_blocks: u64, + block_range: Option<(u64, u64)>, + ) -> Result> { + let bucket_blocks = bucket_blocks.max(1); + self.run_read(move |conn| { + let registry_loaded = verification_registry_loaded(conn, chain_id)?; + let filter = range_filter(block_range); + let has_enrichment = table_exists(conn, "enrichment")?; + let checked_select = if has_enrichment { + format!( + r#" + SELECT + (block_number // {bucket_blocks})::UBIGINT AS bucket_id, + COUNT(*) FILTER (WHERE is_verified)::UBIGINT AS verified, + COUNT(*) FILTER (WHERE is_verified IS FALSE)::UBIGINT AS unverified + FROM enrichment + WHERE block_number IS NOT NULL + AND chain_id = {chain_id} + {filter} + GROUP BY bucket_id + "# + ) + } else { + r#" + SELECT + CAST(NULL AS UBIGINT) AS bucket_id, + 0::UBIGINT AS verified, + 0::UBIGINT AS unverified + WHERE FALSE + "# + .to_string() + }; + + // With a registry loaded, every unchecked deployment is known + // unverified; without one, unchecked deployments are "unknown". + let verified_value = "LEAST(COALESCE(checked.verified, 0), totals.total)"; + let (unverified_expr, unknown_expr) = if registry_loaded { + ( + format!("GREATEST(totals.total - {verified_value}, 0)::UBIGINT"), + "0::UBIGINT".to_string(), + ) + } else { + ( + "COALESCE(checked.unverified, 0)::UBIGINT".to_string(), + format!( + "GREATEST(totals.total - {verified_value} - COALESCE(checked.unverified, 0), 0)::UBIGINT" + ), + ) + }; + let sql = format!( + r#" + WITH totals AS ( + SELECT + (block_number // {bucket_blocks})::UBIGINT AS bucket_id, + SUM(contract_count)::UBIGINT AS total + FROM rollup_block_counts + WHERE chain_id = {chain_id} + {filter} + GROUP BY bucket_id + ), + checked AS ( + {checked_select} + ) + SELECT + totals.bucket_id, + {verified_value}::UBIGINT AS verified, + {unverified_expr} AS unverified, + {unknown_expr} AS unknown + FROM totals + LEFT JOIN checked ON totals.bucket_id = checked.bucket_id + ORDER BY totals.bucket_id + "# + ); + let mut stmt = conn.prepare(&sql)?; + let rows = stmt.query_map([], read_verified_bucket_row)?; + let mut out = Vec::new(); + for r in rows { + let (bucket_id, verified, unverified, unknown) = r?; + let block_start = bucket_id * bucket_blocks; + let block_end = block_start + bucket_blocks - 1; + let mid = block_start + bucket_blocks / 2; + out.push(VerifiedRatioBucket { + block_start, + block_end, + timestamp: crate::blocks::block_timestamp(chain_id, mid), + verified, + unverified, + unknown, + }); + } + Ok(out) + }) + .await + } + + pub async fn bytecode_size_distribution( + &self, + chain_id: u64, + block_range: Option<(u64, u64)>, + ) -> Result> { + self.run_read(move |conn| { + // Fixed semantic buckets keep the heavy tail readable and stop the + // first 1KB range from hiding zero-byte and minimal-proxy contracts. + let bucket_defs: [(u64, u64, &str); 12] = [ + (0, 0, "0 B"), + (1, 32, "1-32 B"), + (33, 44, "33-44 B"), + (45, 45, "45 B minimal proxy"), + (45, 45, "45 B other"), + (46, 64, "46-64 B"), + (65, 256, "65-256 B"), + (257, 1_024, "257 B-1 KB"), + (1_025, 4_096, "1-4 KB"), + (4_097, 8_192, "4-8 KB"), + (8_193, 16_384, "8-16 KB"), + (16_385, 24_576, "16-24 KB"), + ]; + let mut counts = vec![0u64; bucket_defs.len()]; + let bin_case = r#" + CASE + WHEN n_code_bytes = 0 THEN 0 + WHEN n_code_bytes <= 32 THEN 1 + WHEN n_code_bytes <= 44 THEN 2 + WHEN n_code_bytes = 45 AND COALESCE(is_proxy_minimal, false) THEN 3 + WHEN n_code_bytes = 45 THEN 4 + WHEN n_code_bytes <= 64 THEN 5 + WHEN n_code_bytes <= 256 THEN 6 + WHEN n_code_bytes <= 1024 THEN 7 + WHEN n_code_bytes <= 4096 THEN 8 + WHEN n_code_bytes <= 8192 THEN 9 + WHEN n_code_bytes <= 16384 THEN 10 + ELSE 11 + END AS bin_id + "#; + let sql = if explorer_backed(conn)? { + let rows_source = explorer_code_rows_source(chain_id, block_range); + format!( + r#" + SELECT {bin_case}, COUNT(*)::UBIGINT AS cnt + FROM ({rows_source}) rows + WHERE n_code_bytes IS NOT NULL + AND n_code_bytes <= 24576 + GROUP BY bin_id + ORDER BY bin_id + "# + ) + } else { + let count_source = code_counts_source(chain_id, block_range); + format!( + r#" + WITH counts AS ( + {count_source} + ) + SELECT + {bin_case}, + SUM(counts.contract_count)::UBIGINT AS cnt + FROM ( + SELECT + counts.n_code_bytes AS n_code_bytes, + m.is_proxy_minimal AS is_proxy_minimal, + counts.contract_count AS contract_count + FROM counts + LEFT JOIN bytecode_metadata_by_hash m ON counts.code_hash = m.code_hash + ) counts + WHERE counts.n_code_bytes IS NOT NULL + AND counts.n_code_bytes <= 24576 + GROUP BY bin_id + ORDER BY bin_id + "# + ) + }; + let mut stmt = conn.prepare(&sql)?; + let rows = stmt.query_map([], read_u64_pair)?; + for r in rows { + let (bin_id, count) = r?; + if let Some(slot) = counts.get_mut(bin_id as usize) { + *slot = count; + } + } + Ok(bucket_defs + .into_iter() + .enumerate() + .map(|(i, (size_min, size_max, label))| SizeBin { + label: label.to_string(), + size_min, + size_max, + count: counts[i], + }) + .collect()) + }) + .await + } + + pub async fn top_compilers( + &self, + chain_id: u64, + limit: u32, + block_range: Option<(u64, u64)>, + ) -> Result> { + let limit = limit.clamp(1, 50); + self.run_read(move |conn| { + // Compiler distribution is bytecode-derived. Verification sources + // can confirm source publication, but local decode remains the + // source of truth for compiler metadata in this dashboard. + let sql = if explorer_backed(conn)? { + let rows_source = explorer_code_rows_source(chain_id, block_range); + format!( + r#" + SELECT compiler_version, COUNT(*)::UBIGINT AS cnt + FROM ({rows_source}) rows + WHERE compiler_version IS NOT NULL + GROUP BY compiler_version + ORDER BY cnt DESC + LIMIT ? + "# + ) + } else { + let count_source = code_counts_source(chain_id, block_range); + format!( + r#" + WITH counts AS ( + {count_source} + ) + SELECT m.compiler_version, SUM(c.contract_count)::UBIGINT AS cnt + FROM counts c + JOIN bytecode_metadata_by_hash m ON c.code_hash = m.code_hash + WHERE m.compiler_version IS NOT NULL + GROUP BY m.compiler_version + ORDER BY cnt DESC + LIMIT ? + "# + ) + }; + let mut stmt = conn.prepare(&sql)?; + let rows = stmt.query_map(params![limit as i64], read_string_u64_pair)?; + let mut out = Vec::new(); + for r in rows { + let (compiler_version, count) = r?; + out.push(CompilerCount { + compiler_version, + count, + }); + } + Ok(out) + }) + .await + } + + pub async fn compiler_version_total( + &self, + chain_id: u64, + block_range: Option<(u64, u64)>, + ) -> Result { + self.run_read(move |conn| { + let sql = if explorer_backed(conn)? { + let rows_source = explorer_code_rows_source(chain_id, block_range); + format!( + r#" + SELECT COUNT(*)::BIGINT + FROM ({rows_source}) rows + WHERE compiler_version IS NOT NULL + "# + ) + } else { + let count_source = code_counts_source(chain_id, block_range); + format!( + r#" + WITH counts AS ( + {count_source} + ) + SELECT COALESCE(SUM(c.contract_count), 0)::BIGINT + FROM counts c + JOIN bytecode_metadata_by_hash m ON c.code_hash = m.code_hash + WHERE m.compiler_version IS NOT NULL + "# + ) + }; + let count: i64 = conn.query_row(&sql, [], |row| row.get(0)).unwrap_or(0); + Ok(count.max(0) as u64) + }) + .await + } + + pub async fn language_distribution(&self, chain_id: u64) -> Result> { + self.run_read(move |conn| { + let sql = if explorer_backed(conn)? { + let rows_source = explorer_code_rows_source(chain_id, None); + format!( + r#" + SELECT COALESCE(language, 'unknown') AS lang, + COUNT(*)::UBIGINT AS cnt + FROM ({rows_source}) rows + GROUP BY lang + ORDER BY cnt DESC + "# + ) + } else { + format!( + r#" + SELECT COALESCE(m.language, 'unknown') AS lang, + SUM(c.contract_count)::UBIGINT AS cnt + FROM rollup_code_counts c + LEFT JOIN bytecode_metadata_by_hash m ON c.code_hash = m.code_hash + WHERE c.chain_id = {chain_id} + GROUP BY lang + ORDER BY cnt DESC + "# + ) + }; + let mut stmt = conn.prepare(&sql)?; + let rows = stmt.query_map([], read_string_u64_pair)?; + let mut out = Vec::new(); + for r in rows { + let (language, count) = r?; + out.push(LanguageCount { language, count }); + } + Ok(out) + }) + .await + } + + pub async fn standards_breakdown( + &self, + chain_id: u64, + block_range: Option<(u64, u64)>, + ) -> Result { + self.run_read(move |conn| { + let sql = if explorer_backed(conn)? { + let rows_source = explorer_code_rows_source(chain_id, block_range); + format!( + r#" + SELECT + COUNT(*) FILTER (WHERE is_erc20)::UBIGINT, + COUNT(*) FILTER (WHERE is_erc721)::UBIGINT, + COUNT(*) FILTER (WHERE is_erc1155)::UBIGINT, + COUNT(*) FILTER (WHERE is_proxy_eip1967)::UBIGINT, + COUNT(*) FILTER (WHERE is_proxy_minimal)::UBIGINT, + COUNT(*) FILTER (WHERE uses_push0)::UBIGINT, + COUNT(*) FILTER (WHERE has_source_hash)::UBIGINT, + COUNT(*) FILTER (WHERE is_decoded)::UBIGINT + FROM ({rows_source}) rows + "# + ) + } else { + let count_source = code_counts_source(chain_id, block_range); + format!( + r#" + WITH counts AS ( + {count_source} + ) + SELECT + COALESCE(SUM(CASE WHEN m.is_erc20 THEN c.contract_count ELSE 0 END), 0)::UBIGINT, + COALESCE(SUM(CASE WHEN m.is_erc721 THEN c.contract_count ELSE 0 END), 0)::UBIGINT, + COALESCE(SUM(CASE WHEN m.is_erc1155 THEN c.contract_count ELSE 0 END), 0)::UBIGINT, + COALESCE(SUM(CASE WHEN m.is_proxy_eip1967 THEN c.contract_count ELSE 0 END), 0)::UBIGINT, + COALESCE(SUM(CASE WHEN m.is_proxy_minimal THEN c.contract_count ELSE 0 END), 0)::UBIGINT, + COALESCE(SUM(CASE WHEN m.uses_push0 THEN c.contract_count ELSE 0 END), 0)::UBIGINT, + COALESCE(SUM(CASE WHEN m.has_source_hash THEN c.contract_count ELSE 0 END), 0)::UBIGINT, + COALESCE(SUM(CASE WHEN m.code_hash IS NOT NULL THEN c.contract_count ELSE 0 END), 0)::UBIGINT + FROM counts c + LEFT JOIN bytecode_metadata_by_hash m ON c.code_hash = m.code_hash + "# + ) + }; + conn.query_row(&sql, [], |row| { + Ok(StandardsBreakdown { + erc20: row.get(0)?, + erc721: row.get(1)?, + erc1155: row.get(2)?, + proxy_eip1967: row.get(3)?, + proxy_minimal: row.get(4)?, + uses_push0: row.get(5)?, + has_source_hash: row.get(6)?, + total_decoded: row.get(7)?, + }) + }) + .or_else(|_| Ok(StandardsBreakdown::default())) + }) + .await + } + + pub async fn recent_contracts( + &self, + chain_id: u64, + limit: u32, + cursor: Option, + ) -> Result { + let limit = limit.clamp(1, 200); + self.run_read(move |conn| { + let page_limit = limit as usize + 1; + let registry_loaded = verification_registry_loaded(conn, chain_id)?; + + // Recent pages live near the head of the chain: try a bounded + // window first so the top-N scan prunes to a handful of row + // groups, then widen only if the page didn't fill. + let upper = match cursor { + Some(cursor) => Some(cursor.block_number), + None => max_indexed_block(conn, chain_id)?, + }; + let floor = upper.map(|top| top.saturating_sub(RECENT_SCAN_WINDOW_BLOCKS)); + let mut rows = + recent_page_rows(conn, chain_id, cursor, page_limit, floor.filter(|f| *f > 0))?; + if rows.len() < page_limit && floor.map(|f| f > 0).unwrap_or(false) { + rows = recent_page_rows(conn, chain_id, cursor, page_limit, None)?; + } + + let has_more = rows.len() > limit as usize; + rows.truncate(limit as usize); + + // Decorate the page with indexed point lookups instead of hash + // joins — a join rescans the multi-million-row enrichment and + // metadata tables for a 20-row page. + let enrichment = recent_enrichment_by_address(conn, chain_id, &rows)?; + let compilers = recent_compilers_by_hash(conn, &rows)?; + + let contracts = rows + .into_iter() + .map(|row| { + let block_u64 = u64::from(row.block_number); + let (verified, contract_name) = enrichment + .get(&row.address) + .cloned() + .unwrap_or((None, None)); + let is_verified = if registry_loaded { + Some(verified.unwrap_or(false)) + } else { + verified + }; + let compiler_version = row + .code_hash + .as_ref() + .and_then(|hash| compilers.get(hash).cloned()) + .flatten(); + RecentContract { + address: format!("0x{}", hex::encode(&row.address)), + block_number: block_u64, + create_index: u64::from(row.create_index), + timestamp: crate::blocks::block_timestamp(chain_id, block_u64), + deployer: format!("0x{}", hex::encode(row.deployer.unwrap_or_default())), + n_code_bytes: row.n_code_bytes.map(u64::from).unwrap_or(0), + is_verified, + contract_name, + compiler_version, + } + }) + .collect(); + Ok(RecentPage { + contracts, + has_more, + }) + }) + .await + } + + /// Highest block with an indexed contract (rollup MAX — native and cheap). + pub async fn highest_contract_block(&self, chain_id: u64) -> Result> { + self.run_read(move |conn| max_indexed_block(conn, chain_id)) + .await + } + + /// Highest block covered by the dataset, whether or not it contained a + /// contract: max of the parquet filename ranges and the indexed rollup. + /// The tail loop resumes from here. + pub async fn highest_block(&self, chain_id: u64) -> Result> { + let data_dir = self.data_dir().to_path_buf(); + let contracts_glob = self.contracts_glob().to_string(); + self.run_read(move |conn| { + let files = rollups::list_contract_parquet_files(&data_dir, &contracts_glob)?; + let filename_max = rollups::max_contract_file_block_for_chain(&files, chain_id); + let indexed_max = max_indexed_block(conn, chain_id)?; + Ok([filename_max, indexed_max].into_iter().flatten().max()) + }) + .await + } +} + +fn recent_page_rows( + conn: &Connection, + chain_id: u64, + cursor: Option, + page_limit: usize, + floor: Option, +) -> Result> { + let cursor_filter = cursor + .map(|c| { + format!( + "AND (block_number < {block} OR (block_number = {block} AND create_index < {idx}))", + block = c.block_number, + idx = c.create_index + ) + }) + .unwrap_or_default(); + let floor_filter = floor + .map(|f| format!("AND block_number >= {f}")) + .unwrap_or_default(); + let sql = format!( + r#" + SELECT contract_address, block_number, create_index, deployer, n_code_bytes, code_hash + FROM contract_deployments_native + WHERE chain_id = {chain_id} + {floor_filter} + {cursor_filter} + ORDER BY block_number DESC, create_index DESC + LIMIT {page_limit} + "# + ); + let mut stmt = conn.prepare(&sql)?; + let rows = stmt.query_map([], |row| { + Ok(RecentPageRow { + address: row.get(0)?, + block_number: row.get(1)?, + create_index: row.get(2)?, + deployer: row.get(3)?, + n_code_bytes: row.get(4)?, + code_hash: row.get(5)?, + }) + })?; + rows.collect::, _>>() + .context("read recent contracts page") +} + +fn blob_literal(bytes: &[u8]) -> String { + format!("unhex('{}')", hex::encode(bytes)) +} + +/// `(is_verified, contract_name)` per address. +type EnrichmentByAddress = HashMap, (Option, Option)>; + +/// Verification status per address for one page: a UNION ALL of indexed +/// point probes (`enrichment_addr_idx`), ~ms even against millions of rows. +fn recent_enrichment_by_address( + conn: &Connection, + chain_id: u64, + rows: &[RecentPageRow], +) -> Result { + let addresses: HashSet<&Vec> = rows.iter().map(|row| &row.address).collect(); + if addresses.is_empty() { + return Ok(HashMap::new()); + } + let probes = addresses + .iter() + .map(|address| { + format!( + "SELECT contract_address, is_verified, contract_name FROM enrichment_current \ + WHERE contract_address = {} AND chain_id = {chain_id}", + blob_literal(address) + ) + }) + .collect::>() + .join("\nUNION ALL\n"); + let mut stmt = conn.prepare(&probes)?; + let rows = stmt.query_map([], |row| { + Ok(( + row.get::<_, Vec>(0)?, + row.get::<_, Option>(1)?, + row.get::<_, Option>(2)?, + )) + })?; + let mut out = HashMap::new(); + for row in rows { + let (address, is_verified, contract_name) = row?; + out.insert(address, (is_verified, contract_name)); + } + Ok(out) +} + +/// Compiler version per code hash for one page, via +/// `bytecode_metadata_hash_idx` point probes. +fn recent_compilers_by_hash( + conn: &Connection, + rows: &[RecentPageRow], +) -> Result, Option>> { + if !table_exists(conn, "bytecode_metadata_by_hash")? { + return Ok(HashMap::new()); + } + let hashes: HashSet<&Vec> = rows + .iter() + .filter_map(|row| row.code_hash.as_ref()) + .collect(); + if hashes.is_empty() { + return Ok(HashMap::new()); + } + let probes = hashes + .iter() + .map(|hash| { + format!( + "SELECT code_hash, compiler_version FROM bytecode_metadata_by_hash \ + WHERE code_hash = {}", + blob_literal(hash) + ) + }) + .collect::>() + .join("\nUNION ALL\n"); + let mut stmt = conn.prepare(&probes)?; + let rows = stmt.query_map([], |row| { + Ok((row.get::<_, Vec>(0)?, row.get::<_, Option>(1)?)) + })?; + let mut out = HashMap::new(); + for row in rows { + let (hash, compiler_version) = row?; + out.insert(hash, compiler_version); + } + Ok(out) +} diff --git a/src/db/rollups.rs b/src/db/rollups.rs new file mode 100644 index 0000000..bebeaed --- /dev/null +++ b/src/db/rollups.rs @@ -0,0 +1,682 @@ +//! Native rollup tables that make dashboard queries parquet-free. + +use std::{ + collections::HashSet, + path::{Path, PathBuf}, +}; + +use anyhow::{bail, Context, Result}; +use duckdb::{params, Connection}; + +use crate::{chains::ETHEREUM_CHAIN_ID, util::match_simple_glob}; + +use super::{column_exists, table_exists}; + +const ZELLIC_SOURCE_KEY: &str = "zellic://contracts"; + +/// Block-bucket width of `rollup_code_block_counts`. Must match between +/// ingest and the ranged queries that read it. +pub(crate) const CODE_ROLLUP_BUCKET_BLOCKS: u64 = 10_000; + +pub(crate) fn list_contract_parquet_files( + data_dir: &Path, + contracts_glob: &str, +) -> Result> { + let mut files: Vec = std::fs::read_dir(data_dir) + .with_context(|| format!("read data dir {}", data_dir.display()))? + .filter_map(|e| e.ok()) + .map(|e| e.path()) + .filter(|p| { + p.extension().and_then(|s| s.to_str()) == Some("parquet") + && match_simple_glob( + contracts_glob, + p.file_name().and_then(|s| s.to_str()).unwrap_or_default(), + ) + && p.file_name() + .and_then(|s| s.to_str()) + .map(|n| n != "enrichment.parquet") + .unwrap_or(true) + }) + .collect(); + files.sort(); + Ok(files) +} + +#[derive(Debug, Clone, Copy)] +pub(crate) struct ContractFileRange { + pub chain_id: Option, + pub end_block: Option, +} + +/// Chain id and block range hints encoded in a contract parquet file name +/// (`contracts__chain_X__START__END.parquet`, `tail__chain_X__...`, or the +/// legacy chain-less `contracts__START__END.parquet`). +pub(crate) fn contract_file_range(path: &Path) -> ContractFileRange { + let name = path + .file_name() + .and_then(|s| s.to_str()) + .unwrap_or_default(); + let nums = decimal_runs(name); + let chain_id = if name.starts_with("contracts__chain_") || name.starts_with("tail__chain_") { + nums.first().copied() + } else { + None + }; + let end_block = if nums.len() >= 2 { + nums.last().copied() + } else { + None + }; + ContractFileRange { + chain_id, + end_block, + } +} + +fn decimal_runs(input: &str) -> Vec { + let mut out = Vec::new(); + let mut current: Option = None; + for byte in input.bytes() { + if byte.is_ascii_digit() { + let digit = u64::from(byte - b'0'); + current = Some( + current + .unwrap_or(0) + .saturating_mul(10) + .saturating_add(digit), + ); + } else if let Some(value) = current.take() { + out.push(value); + } + } + if let Some(value) = current { + out.push(value); + } + out +} + +pub(crate) fn max_contract_file_block_for_chain(files: &[PathBuf], chain_id: u64) -> Option { + files + .iter() + .map(|path| contract_file_range(path)) + .filter(|file| { + file.chain_id == Some(chain_id) + || (file.chain_id.is_none() && chain_id == ETHEREUM_CHAIN_ID) + }) + .filter_map(|file| file.end_block) + .max() +} + +pub(crate) fn ensure_rollup_schema(conn: &Connection) -> Result<()> { + conn.execute_batch( + r#" + CREATE TABLE IF NOT EXISTS contract_deployments_native ( + chain_id UBIGINT NOT NULL, + block_number UINTEGER NOT NULL, + create_index UINTEGER NOT NULL, + contract_address BLOB NOT NULL, + deployer BLOB, + code_hash BLOB, + n_code_bytes UINTEGER, + source_path VARCHAR + ); + CREATE TABLE IF NOT EXISTS rollup_block_counts ( + chain_id UBIGINT NOT NULL, + block_number UINTEGER NOT NULL, + contract_count UBIGINT NOT NULL, + PRIMARY KEY (chain_id, block_number) + ); + CREATE TABLE IF NOT EXISTS rollup_code_counts ( + chain_id UBIGINT NOT NULL, + code_hash BLOB NOT NULL, + n_code_bytes UINTEGER, + contract_count UBIGINT NOT NULL, + PRIMARY KEY (chain_id, code_hash) + ); + CREATE TABLE IF NOT EXISTS rollup_code_block_counts ( + chain_id UBIGINT NOT NULL, + block_bucket UBIGINT NOT NULL, + code_hash BLOB NOT NULL, + n_code_bytes UINTEGER, + contract_count UBIGINT NOT NULL, + PRIMARY KEY (chain_id, block_bucket, code_hash) + ); + CREATE TABLE IF NOT EXISTS rollup_sources ( + source_path VARCHAR PRIMARY KEY, + start_block UBIGINT, + end_block UBIGINT, + row_count UBIGINT NOT NULL, + ingested_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ); + -- Superseded: per-source counts double-counted overlapping files. + DROP TABLE IF EXISTS parquet_block_counts; + "#, + ) + .context("create rollup schema") +} + +pub(crate) fn rollups_ready(conn: &Connection) -> Result { + Ok(table_exists(conn, "contract_deployments_native")? + && table_exists(conn, "rollup_block_counts")? + && table_exists(conn, "rollup_code_counts")? + && table_exists(conn, "rollup_code_block_counts")? + && table_exists(conn, "rollup_sources")?) +} + +/// One-time migration: databases built before the bucketed code rollup +/// existed have deployments but an empty `rollup_code_block_counts`. +fn backfill_code_block_rollup(conn: &Connection) -> Result<()> { + let (bucket_rows, deployment_rows): (i64, i64) = conn.query_row( + r#" + SELECT + (SELECT COUNT(*) FROM rollup_code_block_counts), + (SELECT COUNT(*) FROM contract_deployments_native) + "#, + [], + |row| Ok((row.get(0)?, row.get(1)?)), + )?; + if bucket_rows > 0 || deployment_rows == 0 { + return Ok(()); + } + tracing::info!("backfilling bucketed code rollup from native deployments (one-time)"); + conn.execute_batch(&format!( + r#" + INSERT INTO rollup_code_block_counts + SELECT + chain_id, + (block_number // {CODE_ROLLUP_BUCKET_BLOCKS})::UBIGINT, + code_hash, + any_value(n_code_bytes)::UINTEGER, + COUNT(*)::UBIGINT + FROM contract_deployments_native + WHERE code_hash IS NOT NULL + GROUP BY chain_id, (block_number // {CODE_ROLLUP_BUCKET_BLOCKS}), code_hash; + "# + )) + .context("backfill bucketed code rollup") +} + +/// Bring the rollups in line with the current parquet file set (plus the +/// optional Zellic snapshot tables). Returns whether anything was ingested. +pub(crate) fn sync_rollups( + conn: &Connection, + data_dir: &Path, + contracts_glob: &str, +) -> Result { + backfill_code_block_rollup(conn)?; + let files = list_contract_parquet_files(data_dir, contracts_glob)?; + let mut tracked = tracked_sources(conn)?; + + let mut current: HashSet = files.iter().map(|p| p.display().to_string()).collect(); + let has_zellic = table_exists(conn, "zellic_contracts")?; + if has_zellic { + current.insert(ZELLIC_SOURCE_KEY.to_string()); + } + + let removed: Vec = tracked.difference(¤t).cloned().collect(); + if !removed.is_empty() { + tracing::warn!( + "{} rollup source(s) disappeared (e.g. {}); rebuilding deployment rollups from scratch", + removed.len(), + removed[0] + ); + conn.execute_batch( + r#" + BEGIN; + DELETE FROM contract_deployments_native; + DELETE FROM rollup_block_counts; + DELETE FROM rollup_code_counts; + DELETE FROM rollup_code_block_counts; + DELETE FROM rollup_sources; + COMMIT; + "#, + ) + .context("reset deployment rollups")?; + tracked.clear(); + } + + let mut changed = !removed.is_empty(); + for file in &files { + let key = file.display().to_string(); + if tracked.contains(&key) { + continue; + } + match ingest_parquet(conn, file) { + Ok(rows) => { + changed = true; + tracing::info!( + "rolled up {} ({} new deployments)", + file.file_name() + .and_then(|n| n.to_str()) + .unwrap_or(key.as_str()), + rows + ); + } + Err(err) => { + tracing::warn!("skipping rollup for {}: {:#}", file.display(), err); + } + } + } + if has_zellic && !tracked.contains(ZELLIC_SOURCE_KEY) { + tracing::info!("rolling up Zellic snapshot into native deployments (one-time)"); + match ingest_zellic(conn) { + Ok(rows) => { + tracing::info!("zellic rollup complete ({} new deployments)", rows); + changed = true; + } + // Not fatal: the dashboard can serve the parquet-era data while + // the operator fixes this; the next start retries and already- + // committed slices are skipped by dedup. + Err(err) => tracing::error!( + "zellic rollup failed; serving without zellic history until the next successful sync: {:#}", + err + ), + } + } + + Ok(changed) +} + +fn tracked_sources(conn: &Connection) -> Result> { + let mut stmt = conn.prepare("SELECT source_path FROM rollup_sources")?; + let rows = stmt.query_map([], |row| row.get::<_, String>(0))?; + rows.collect::, _>>() + .context("list tracked rollup sources") +} + +/// Column names of a parquet file, so ingest can adapt to the differing +/// schemas of blink/cryo/paradigm sources instead of failing outright. +fn parquet_columns(conn: &Connection, path_sql: &str) -> Result> { + let mut stmt = conn.prepare(&format!( + "DESCRIBE SELECT * FROM read_parquet('{path_sql}')" + ))?; + let rows = stmt.query_map([], |row| row.get::<_, String>(0))?; + rows.collect::, _>>() + .context("describe parquet columns") +} + +fn ingest_parquet(conn: &Connection, path: &Path) -> Result { + let source_key = path.display().to_string(); + let path_sql = source_key.replace('\'', "''"); + let columns = parquet_columns(conn, &path_sql)?; + if !columns.contains("block_number") || !columns.contains("contract_address") { + bail!("parquet file lacks block_number/contract_address columns"); + } + + let fallback_chain = contract_file_range(path) + .chain_id + .unwrap_or(ETHEREUM_CHAIN_ID); + let chain_expr = if columns.contains("chain_id") { + format!("COALESCE(chain_id, {fallback_chain})") + } else { + fallback_chain.to_string() + }; + let create_index_expr = if columns.contains("create_index") { + "COALESCE(create_index, 0)" + } else { + "0" + }; + let deployer_expr = if columns.contains("deployer") { + "deployer" + } else { + "CAST(NULL AS BLOB)" + }; + let code_hash_expr = if columns.contains("code_hash") { + "code_hash" + } else { + "CAST(NULL AS BLOB)" + }; + let n_code_bytes_expr = if columns.contains("n_code_bytes") { + "n_code_bytes" + } else if columns.contains("code") { + "length(code)" + } else { + "CAST(NULL AS UINTEGER)" + }; + + let select = format!( + r#" + SELECT + {chain_expr}::UBIGINT AS chain_id, + block_number::UINTEGER AS block_number, + {create_index_expr}::UINTEGER AS create_index, + contract_address, + {deployer_expr} AS deployer, + {code_hash_expr} AS code_hash, + {n_code_bytes_expr}::UINTEGER AS n_code_bytes + FROM read_parquet('{path_sql}') + WHERE block_number IS NOT NULL + AND contract_address IS NOT NULL + "# + ); + let (inserted, min_block, max_block) = ingest_rows(conn, &source_key, &select)?; + record_source(conn, &source_key, min_block, max_block, inserted)?; + Ok(inserted) +} + +/// Rows per zellic ingest slice. The snapshot is tens of millions of rows; +/// ingesting it in one transaction OOMs small hosts (join hash table + +/// transaction state), so it is sliced by block range into transactions of +/// roughly this size. Dedup makes re-running a slice a no-op, so a crash +/// mid-snapshot resumes cleanly on the next start. +const ZELLIC_SLICE_TARGET_ROWS: u64 = 1_000_000; + +fn ingest_zellic(conn: &Connection) -> Result { + let chain_expr = if column_exists(conn, "zellic_contracts", "chain_id")? { + "COALESCE(z.chain_id, 1)" + } else { + "1" + }; + let (bytecode_join, n_code_bytes_expr) = if table_exists(conn, "zellic_bytecodes")? { + ( + "LEFT JOIN zellic_bytecodes b ON z.bytecode_hash = b.code_hash", + "b.n_code_bytes", + ) + } else { + ("", "CAST(NULL AS UINTEGER)") + }; + + let (min_block, max_block, total_rows): (Option, Option, i64) = conn + .query_row( + r#" + SELECT MIN(block_number), MAX(block_number), COUNT(*) + FROM zellic_contracts + WHERE block_number IS NOT NULL AND contract_address IS NOT NULL + "#, + [], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .context("measure zellic snapshot")?; + let (Some(min_block), Some(max_block)) = (min_block, max_block) else { + record_source(conn, ZELLIC_SOURCE_KEY, None, None, 0)?; + return Ok(0); + }; + let (min_block, max_block) = (u64::from(min_block), u64::from(max_block)); + + let span = max_block - min_block + 1; + let slice_count = (total_rows.max(0) as u64) + .div_ceil(ZELLIC_SLICE_TARGET_ROWS) + .max(1); + let slice_blocks = span.div_ceil(slice_count).max(1); + let total_slices = span.div_ceil(slice_blocks); + + let mut inserted = 0u64; + let mut slice_start = min_block; + let mut slice_index = 0u64; + while slice_start <= max_block { + let slice_end = (slice_start + slice_blocks - 1).min(max_block); + slice_index += 1; + let select = format!( + r#" + SELECT + {chain_expr}::UBIGINT AS chain_id, + z.block_number::UINTEGER AS block_number, + COALESCE(z.create_index, 0)::UINTEGER AS create_index, + z.contract_address, + CAST(NULL AS BLOB) AS deployer, + z.bytecode_hash AS code_hash, + {n_code_bytes_expr}::UINTEGER AS n_code_bytes + FROM zellic_contracts z + {bytecode_join} + WHERE z.block_number IS NOT NULL + AND z.contract_address IS NOT NULL + AND z.block_number BETWEEN {slice_start} AND {slice_end} + "# + ); + let (slice_rows, _, _) = ingest_rows(conn, ZELLIC_SOURCE_KEY, &select) + .with_context(|| format!("zellic slice blocks {slice_start}-{slice_end}"))?; + inserted += slice_rows; + if total_slices > 1 { + tracing::info!( + "zellic rollup progress: slice {}/{} (blocks {}-{}, {} deployments so far)", + slice_index, + total_slices, + slice_start, + slice_end, + inserted + ); + } + slice_start = slice_end + 1; + } + + record_source( + conn, + ZELLIC_SOURCE_KEY, + Some(min_block as u32), + Some(max_block as u32), + inserted, + )?; + Ok(inserted) +} + +/// Mark a source as ingested. Separate from [`ingest_rows`] so a source can +/// be ingested in several slice transactions and recorded only once at the +/// end — a crash in between just means the committed rows get re-offered and +/// deduplicated away on the next start. +fn record_source( + conn: &Connection, + source_key: &str, + min_block: Option, + max_block: Option, + inserted: u64, +) -> Result<()> { + conn.execute( + "INSERT INTO rollup_sources (source_path, start_block, end_block, row_count) VALUES (?, ?, ?, ?)", + params![ + source_key, + min_block.map(u64::from), + max_block.map(u64::from), + inserted + ], + ) + .with_context(|| format!("record rollup source {source_key}"))?; + Ok(()) +} + +/// Ingest one batch of rows inside a transaction: dedup the incoming rows +/// within the batch and against everything already indexed for the affected +/// block window, append the survivors, and fold the same delta into the +/// summary tables so they can never drift from the deployments table. +/// Returns (inserted, min_block, max_block) of the batch. +fn ingest_rows( + conn: &Connection, + source_key: &str, + select: &str, +) -> Result<(u64, Option, Option)> { + let source_sql = source_key.replace('\'', "''"); + let result = (|| -> Result<(u64, Option, Option)> { + conn.execute_batch("BEGIN;")?; + conn.execute_batch(&format!( + r#" + CREATE OR REPLACE TEMP TABLE rollup_ingest AS + SELECT + chain_id, + block_number, + create_index, + contract_address, + any_value(deployer) AS deployer, + any_value(code_hash) AS code_hash, + any_value(n_code_bytes) AS n_code_bytes + FROM ({select}) src + GROUP BY chain_id, block_number, create_index, contract_address; + "# + ))?; + let (min_block, max_block): (Option, Option) = conn.query_row( + "SELECT MIN(block_number), MAX(block_number) FROM rollup_ingest", + [], + |row| Ok((row.get(0)?, row.get(1)?)), + )?; + + let mut inserted = 0u64; + if let (Some(min_block), Some(max_block)) = (min_block, max_block) { + conn.execute_batch(&format!( + r#" + DELETE FROM rollup_ingest + WHERE EXISTS ( + SELECT 1 FROM contract_deployments_native c + WHERE c.block_number BETWEEN {min_block} AND {max_block} + AND c.chain_id = rollup_ingest.chain_id + AND c.block_number = rollup_ingest.block_number + AND c.create_index = rollup_ingest.create_index + AND c.contract_address = rollup_ingest.contract_address + ); + + INSERT INTO contract_deployments_native + SELECT + chain_id, block_number, create_index, contract_address, + deployer, code_hash, n_code_bytes, '{source_sql}' + FROM rollup_ingest; + + INSERT INTO rollup_block_counts + SELECT chain_id, block_number, COUNT(*)::UBIGINT + FROM rollup_ingest + GROUP BY chain_id, block_number + ON CONFLICT (chain_id, block_number) + DO UPDATE SET contract_count = contract_count + excluded.contract_count; + + INSERT INTO rollup_code_counts + SELECT chain_id, code_hash, any_value(n_code_bytes)::UINTEGER, COUNT(*)::UBIGINT + FROM rollup_ingest + WHERE code_hash IS NOT NULL + GROUP BY chain_id, code_hash + ON CONFLICT (chain_id, code_hash) + DO UPDATE SET contract_count = contract_count + excluded.contract_count; + + INSERT INTO rollup_code_block_counts + SELECT + chain_id, + (block_number // {CODE_ROLLUP_BUCKET_BLOCKS})::UBIGINT, + code_hash, + any_value(n_code_bytes)::UINTEGER, + COUNT(*)::UBIGINT + FROM rollup_ingest + WHERE code_hash IS NOT NULL + GROUP BY chain_id, (block_number // {CODE_ROLLUP_BUCKET_BLOCKS}), code_hash + ON CONFLICT (chain_id, block_bucket, code_hash) + DO UPDATE SET contract_count = contract_count + excluded.contract_count; + "# + ))?; + inserted = conn.query_row("SELECT COUNT(*) FROM rollup_ingest", [], |row| { + row.get::<_, i64>(0) + })? as u64; + } + + conn.execute_batch("DROP TABLE IF EXISTS rollup_ingest; COMMIT;")?; + Ok((inserted, min_block, max_block)) + })(); + if result.is_err() { + let _ = conn.execute_batch("ROLLBACK;"); + let _ = conn.execute_batch("DROP TABLE IF EXISTS rollup_ingest;"); + } + result.with_context(|| format!("ingest rollup source {source_key}")) +} + +/// Subtract one source's exact contribution from the deployments table and +/// every summary rollup, and forget it in `rollup_sources` so the next sync +/// re-ingests it. Exact because every deployment row records which source +/// inserted it; rows this source lost to dedup belong to another source and +/// correctly stay. +fn invalidate_source(conn: &Connection, source_key: &str) -> Result<()> { + if !rollups_ready(conn)? { + return Ok(()); + } + let source_sql = source_key.replace('\'', "''"); + let result = (|| -> Result<()> { + conn.execute_batch(&format!( + r#" + BEGIN; + CREATE OR REPLACE TEMP TABLE rollup_removed AS + SELECT chain_id, block_number, code_hash, n_code_bytes + FROM contract_deployments_native + WHERE source_path = '{source_sql}'; + + DELETE FROM contract_deployments_native + WHERE source_path = '{source_sql}'; + + UPDATE rollup_block_counts + SET contract_count = rollup_block_counts.contract_count - d.cnt + FROM ( + SELECT chain_id, block_number, COUNT(*)::UBIGINT AS cnt + FROM rollup_removed + GROUP BY chain_id, block_number + ) d + WHERE rollup_block_counts.chain_id = d.chain_id + AND rollup_block_counts.block_number = d.block_number; + DELETE FROM rollup_block_counts WHERE contract_count = 0; + + UPDATE rollup_code_counts + SET contract_count = rollup_code_counts.contract_count - d.cnt + FROM ( + SELECT chain_id, code_hash, COUNT(*)::UBIGINT AS cnt + FROM rollup_removed + WHERE code_hash IS NOT NULL + GROUP BY chain_id, code_hash + ) d + WHERE rollup_code_counts.chain_id = d.chain_id + AND rollup_code_counts.code_hash = d.code_hash; + DELETE FROM rollup_code_counts WHERE contract_count = 0; + + UPDATE rollup_code_block_counts + SET contract_count = rollup_code_block_counts.contract_count - d.cnt + FROM ( + SELECT + chain_id, + (block_number // {CODE_ROLLUP_BUCKET_BLOCKS})::UBIGINT AS block_bucket, + code_hash, + COUNT(*)::UBIGINT AS cnt + FROM rollup_removed + WHERE code_hash IS NOT NULL + GROUP BY chain_id, (block_number // {CODE_ROLLUP_BUCKET_BLOCKS}), code_hash + ) d + WHERE rollup_code_block_counts.chain_id = d.chain_id + AND rollup_code_block_counts.block_bucket = d.block_bucket + AND rollup_code_block_counts.code_hash = d.code_hash; + DELETE FROM rollup_code_block_counts WHERE contract_count = 0; + + DELETE FROM rollup_sources WHERE source_path = '{source_sql}'; + DROP TABLE IF EXISTS rollup_removed; + COMMIT; + "# + ))?; + Ok(()) + })(); + if result.is_err() { + let _ = conn.execute_batch("ROLLBACK;"); + let _ = conn.execute_batch("DROP TABLE IF EXISTS rollup_removed;"); + } + result.with_context(|| format!("invalidate rollup source {source_key}")) +} + +/// Called by `blink load --overwrite` before it drops and re-imports the +/// Zellic snapshot tables: without this, the rebuilt snapshot would never be +/// re-rolled-up because `rollup_sources` still marks it as ingested. +pub fn invalidate_zellic_rollups(conn: &Connection) -> Result<()> { + invalidate_source(conn, ZELLIC_SOURCE_KEY) +} + +/// Fill enrichment rows that lack a block position from the deployments +/// table, so the verified-ratio chart can bucket them without joins at +/// request time. +pub(crate) fn backfill_enrichment_blocks(conn: &Connection) -> Result<()> { + if !table_exists(conn, "enrichment")? { + return Ok(()); + } + if !column_exists(conn, "enrichment", "block_number")? + || !column_exists(conn, "enrichment", "create_index")? + { + return Ok(()); + } + conn.execute_batch( + r#" + UPDATE enrichment AS e + SET + block_number = c.block_number, + create_index = c.create_index + FROM contract_deployments_native AS c + WHERE e.contract_address = c.contract_address + AND e.chain_id = c.chain_id + AND (e.block_number IS NULL OR e.create_index IS NULL); + "#, + ) + .context("backfill enrichment block positions") +} diff --git a/src/db/sql.rs b/src/db/sql.rs new file mode 100644 index 0000000..16704c5 --- /dev/null +++ b/src/db/sql.rs @@ -0,0 +1,171 @@ +//! Read-only SQL guard rails and JSON conversion for `POST /api/query`. + +use anyhow::{anyhow, Result}; +use duckdb::types::ValueRef; +use serde_json::{Number, Value}; + +#[derive(Debug, Clone, serde::Serialize, utoipa::ToSchema)] +pub struct SqlQueryResult { + pub columns: Vec, + #[schema(value_type = Vec>)] + pub rows: Vec>, + pub row_count: usize, + pub limit: u32, + pub elapsed_ms: u128, +} + +fn contains_sql_keyword(sql: &str, keyword: &str) -> bool { + let bytes = sql.as_bytes(); + let needle = keyword.as_bytes(); + if needle.is_empty() || needle.len() > bytes.len() { + return false; + } + + bytes + .windows(needle.len()) + .enumerate() + .any(|(idx, window)| { + if window != needle { + return false; + } + let before = idx.checked_sub(1).and_then(|i| bytes.get(i)).copied(); + let after = bytes.get(idx + needle.len()).copied(); + !is_sql_ident_byte(before) && !is_sql_ident_byte(after) + }) +} + +fn is_sql_ident_byte(byte: Option) -> bool { + matches!(byte, Some(b'a'..=b'z' | b'0'..=b'9' | b'_')) +} + +pub(crate) fn normalize_read_only_sql(sql: &str) -> Result { + let trimmed = sql.trim(); + if trimmed.is_empty() { + return Err(anyhow!("query is empty")); + } + if trimmed.len() > 20_000 { + return Err(anyhow!("query is too large")); + } + + let without_trailing_semicolon = trimmed + .strip_suffix(';') + .map(str::trim_end) + .unwrap_or(trimmed); + if without_trailing_semicolon.contains(';') { + return Err(anyhow!("only one read-only statement is allowed")); + } + + let lower = without_trailing_semicolon.to_ascii_lowercase(); + let first = lower.split_whitespace().next().unwrap_or_default(); + if first != "select" && first != "with" { + return Err(anyhow!("only SELECT and WITH queries are allowed")); + } + + for keyword in [ + "alter", + "attach", + "call", + "checkpoint", + "copy", + "create", + "delete", + "detach", + "drop", + "export", + "import", + "insert", + "install", + "load", + "pragma", + "set", + "update", + "vacuum", + ] { + if contains_sql_keyword(&lower, keyword) { + return Err(anyhow!( + "keyword `{}` is not allowed in dashboard queries", + keyword + )); + } + } + + for function in [ + "read_blob", + "read_csv", + "read_json", + "read_parquet", + "csv_scan", + "parquet_scan", + "sqlite_scan", + "postgres_scan", + "mysql_scan", + "httpfs", + ] { + if lower.contains(function) { + return Err(anyhow!( + "file and extension access is not allowed in dashboard queries" + )); + } + } + + Ok(without_trailing_semicolon.to_string()) +} + +pub(crate) fn wrap_dashboard_query(sql: &str, limit: u32, chain_id: Option) -> String { + match chain_id { + Some(chain_id) => format!( + r#" + WITH contract_metadata AS ( + SELECT * + FROM contract_metadata_all + WHERE chain_id = {chain_id} + ) + SELECT * + FROM ({sql}) AS _blink_dashboard_query + LIMIT {limit} + "# + ), + None => format!("SELECT * FROM ({sql}) AS _blink_dashboard_query LIMIT {limit}"), + } +} + +pub(crate) fn value_ref_to_json(value: ValueRef<'_>) -> Value { + match value { + ValueRef::Null => Value::Null, + ValueRef::Boolean(value) => Value::Bool(value), + ValueRef::TinyInt(value) => Value::Number(Number::from(value)), + ValueRef::SmallInt(value) => Value::Number(Number::from(value)), + ValueRef::Int(value) => Value::Number(Number::from(value)), + ValueRef::BigInt(value) => Value::Number(Number::from(value)), + ValueRef::HugeInt(value) => i64::try_from(value) + .map(Number::from) + .map(Value::Number) + .unwrap_or_else(|_| Value::String(value.to_string())), + ValueRef::UTinyInt(value) => Value::Number(Number::from(value)), + ValueRef::USmallInt(value) => Value::Number(Number::from(value)), + ValueRef::UInt(value) => Value::Number(Number::from(value)), + ValueRef::UBigInt(value) => Value::Number(Number::from(value)), + ValueRef::Float(value) => Number::from_f64(value as f64) + .map(Value::Number) + .unwrap_or(Value::Null), + ValueRef::Double(value) => Number::from_f64(value) + .map(Value::Number) + .unwrap_or(Value::Null), + ValueRef::Decimal(value) => Value::String(value.to_string()), + ValueRef::Timestamp(unit, value) => { + Value::String(format!("{} {:?}", value, unit).to_ascii_lowercase()) + } + ValueRef::Text(value) => Value::String(String::from_utf8_lossy(value).into_owned()), + ValueRef::Blob(value) => Value::String(format!("0x{}", hex::encode(value))), + ValueRef::Date32(value) => Value::Number(Number::from(value)), + ValueRef::Time64(unit, value) => { + Value::String(format!("{} {:?}", value, unit).to_ascii_lowercase()) + } + ValueRef::Interval { + months, + days, + nanos, + } => Value::String(format!("{months} months {days} days {nanos} ns")), + other => Value::String(format!("{other:?}")), + } +} diff --git a/src/db/views.rs b/src/db/views.rs new file mode 100644 index 0000000..7ba03a2 --- /dev/null +++ b/src/db/views.rs @@ -0,0 +1,660 @@ +//! Persistent schema (enrichment + decode metadata tables) and the temp views +//! backing `POST /api/query`. + +use std::path::PathBuf; + +use anyhow::{Context, Result}; +use duckdb::Connection; + +use super::{column_exists, table_exists}; + +/// Create/migrate the persistent tables owned by the serve writer. +pub(crate) fn ensure_schema(conn: &Connection) -> Result<()> { + conn.execute_batch( + r#" + CREATE TABLE IF NOT EXISTS enrichment ( + contract_address BLOB, + chain_id UBIGINT DEFAULT 1, + is_verified BOOLEAN NOT NULL, + contract_name VARCHAR, + checked_at TIMESTAMP NOT NULL + ); + -- Track where each verification came from (verifier_alliance). + -- Added in a later migration; the IF NOT EXISTS guard keeps older + -- databases working without an explicit migration step. + ALTER TABLE enrichment ADD COLUMN IF NOT EXISTS chain_id UBIGINT DEFAULT 1; + ALTER TABLE enrichment ADD COLUMN IF NOT EXISTS verification_source VARCHAR; + ALTER TABLE enrichment ADD COLUMN IF NOT EXISTS match_type VARCHAR; + ALTER TABLE enrichment ADD COLUMN IF NOT EXISTS block_number UINTEGER; + ALTER TABLE enrichment ADD COLUMN IF NOT EXISTS create_index UINTEGER; + UPDATE enrichment SET chain_id = 1 WHERE chain_id IS NULL; + CREATE INDEX IF NOT EXISTS enrichment_chain_addr_idx ON enrichment(chain_id, contract_address); + CREATE INDEX IF NOT EXISTS enrichment_chain_block_idx ON enrichment(chain_id, block_number); + CREATE INDEX IF NOT EXISTS enrichment_verified_idx ON enrichment(is_verified); + CREATE INDEX IF NOT EXISTS enrichment_source_idx ON enrichment(verification_source); + -- Single-column index: DuckDB uses this for the per-address point + -- lookups decorating /api/recent pages (the composite index above is + -- not chosen for them). + CREATE INDEX IF NOT EXISTS enrichment_addr_idx ON enrichment(contract_address); + + CREATE TABLE IF NOT EXISTS bytecode_metadata_v2 ( + contract_address BLOB NOT NULL, + language VARCHAR, + compiler_version VARCHAR, + has_source_hash BOOLEAN NOT NULL, + is_erc20 BOOLEAN NOT NULL, + is_erc721 BOOLEAN NOT NULL, + is_erc1155 BOOLEAN NOT NULL, + is_proxy_eip1967 BOOLEAN NOT NULL, + is_proxy_minimal BOOLEAN NOT NULL DEFAULT false, + uses_push0 BOOLEAN NOT NULL, + source_file VARCHAR NOT NULL, + decoded_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ); + + CREATE TABLE IF NOT EXISTS bytecode_metadata_by_hash ( + code_hash BLOB NOT NULL, + language VARCHAR, + compiler_version VARCHAR, + has_source_hash BOOLEAN NOT NULL, + is_erc20 BOOLEAN NOT NULL, + is_erc721 BOOLEAN NOT NULL, + is_erc1155 BOOLEAN NOT NULL, + is_proxy_eip1967 BOOLEAN NOT NULL, + is_proxy_minimal BOOLEAN NOT NULL DEFAULT false, + uses_push0 BOOLEAN NOT NULL, + decoded_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ); + ALTER TABLE bytecode_metadata_by_hash + ADD COLUMN IF NOT EXISTS decoded_at TIMESTAMP; + -- Point lookups for /api/recent page decoration. + CREATE INDEX IF NOT EXISTS bytecode_metadata_hash_idx + ON bytecode_metadata_by_hash(code_hash); + -- EIP-1167 minimal proxy detection added later; backfill the column + -- with `false` on existing rows. DuckDB cannot add constrained + -- columns to an existing table. + ALTER TABLE bytecode_metadata_v2 + ADD COLUMN IF NOT EXISTS is_proxy_minimal BOOLEAN; + ALTER TABLE bytecode_metadata_by_hash + ADD COLUMN IF NOT EXISTS is_proxy_minimal BOOLEAN; + UPDATE bytecode_metadata_v2 + SET is_proxy_minimal = false + WHERE is_proxy_minimal IS NULL; + UPDATE bytecode_metadata_by_hash + SET is_proxy_minimal = false + WHERE is_proxy_minimal IS NULL; + "#, + ) + .context("create blink schema") +} + +/// Rebuild every temp view on this connection: the parquet-backed raw views +/// plus the metadata/enrichment compatibility views. +pub(crate) fn rebuild_query_views(conn: &Connection, files: &[PathBuf]) -> Result<()> { + rebuild_parquet_views(conn, files)?; + create_metadata_current_view(conn)?; + create_enrichment_current_view(conn)?; + create_standard_query_views(conn)?; + Ok(()) +} + +/// Rebuild only the views whose definition embeds the parquet file list. +/// Cheap; called on every connection when the tail loop lands a new file. +pub(crate) fn rebuild_parquet_views(conn: &Connection, files: &[PathBuf]) -> Result<()> { + let empty_select = r#" + SELECT + CAST(NULL AS UINTEGER) AS block_number, + CAST(NULL AS BLOB) AS block_hash, + CAST(NULL AS UINTEGER) AS create_index, + CAST(NULL AS BLOB) AS transaction_hash, + CAST(NULL AS BLOB) AS contract_address, + CAST(NULL AS BLOB) AS deployer, + CAST(NULL AS BLOB) AS factory, + CAST(NULL AS BLOB) AS init_code, + CAST(NULL AS BLOB) AS code, + CAST(NULL AS BLOB) AS init_code_hash, + CAST(NULL AS UINTEGER) AS n_init_code_bytes, + CAST(NULL AS UINTEGER) AS n_code_bytes, + CAST(NULL AS BLOB) AS code_hash, + CAST(NULL AS UBIGINT) AS chain_id + WHERE FALSE + "#; + + let parquet_body = if files.is_empty() { + empty_select.to_string() + } else { + let list = files + .iter() + .map(|p| format!("'{}'", p.display().to_string().replace('\'', "''"))) + .collect::>() + .join(", "); + format!( + r#" + SELECT + block_number, block_hash, create_index, transaction_hash, + contract_address, deployer, factory, init_code, code, + init_code_hash, n_init_code_bytes, n_code_bytes, + code_hash, chain_id + FROM read_parquet([{list}], union_by_name = true) + "# + ) + }; + conn.execute_batch(&format!( + "CREATE OR REPLACE TEMP VIEW parquet_contracts AS\n{parquet_body};" + )) + .with_context(|| format!("create parquet contracts view ({} files)", files.len()))?; + + let has_zellic = + table_exists(conn, "zellic_contracts")? && table_exists(conn, "zellic_bytecodes")?; + let zellic_select = if has_zellic { + Some( + r#" + SELECT + z.block_number, + CAST(NULL AS BLOB) AS block_hash, + z.create_index, + CAST(NULL AS BLOB) AS transaction_hash, + z.contract_address, + CAST(NULL AS BLOB) AS deployer, + CAST(NULL AS BLOB) AS factory, + CAST(NULL AS BLOB) AS init_code, + b.code, + CAST(NULL AS BLOB) AS init_code_hash, + CAST(NULL AS UINTEGER) AS n_init_code_bytes, + b.n_code_bytes, + z.bytecode_hash AS code_hash, + z.chain_id + FROM zellic_contracts z + LEFT JOIN zellic_bytecodes b ON z.bytecode_hash = b.code_hash + "# + .to_string(), + ) + } else { + None + }; + + let selects = [ + if files.is_empty() { + None + } else { + Some("SELECT * FROM parquet_contracts".to_string()) + }, + zellic_select, + ] + .into_iter() + .flatten() + .collect::>(); + let body = if selects.is_empty() { + empty_select.to_string() + } else { + selects.join("\nUNION ALL\n") + }; + conn.execute_batch(&format!( + "CREATE OR REPLACE TEMP VIEW contracts AS\n{body};" + )) + .with_context(|| format!("create contracts view ({} files)", files.len())) +} + +fn create_empty_metadata_current_view(conn: &Connection) -> Result<()> { + conn.execute_batch( + r#" + CREATE OR REPLACE TEMP VIEW bytecode_metadata_current AS + SELECT + CAST(NULL AS BLOB) AS contract_address, + CAST(NULL AS VARCHAR) AS language, + CAST(NULL AS VARCHAR) AS compiler_version, + CAST(false AS BOOLEAN) AS has_source_hash, + CAST(false AS BOOLEAN) AS is_erc20, + CAST(false AS BOOLEAN) AS is_erc721, + CAST(false AS BOOLEAN) AS is_erc1155, + CAST(false AS BOOLEAN) AS is_proxy_eip1967, + CAST(false AS BOOLEAN) AS is_proxy_minimal, + CAST(false AS BOOLEAN) AS uses_push0, + CAST(NULL AS TIMESTAMP) AS decoded_at + WHERE FALSE; + "#, + ) + .context("create empty metadata view") +} + +fn create_metadata_current_view(conn: &Connection) -> Result<()> { + let has_v1 = table_exists(conn, "bytecode_metadata")?; + let has_v2 = table_exists(conn, "bytecode_metadata_v2")?; + + if !has_v1 && !has_v2 { + return create_empty_metadata_current_view(conn); + } + + let address_meta = match (has_v1, has_v2) { + (false, false) => unreachable!("guarded above"), + (true, false) => { + // v1 predates EIP-1167 detection — synthesize a false column so the + // view shape matches. + r#" + SELECT + contract_address, language, compiler_version, has_source_hash, + is_erc20, is_erc721, is_erc1155, is_proxy_eip1967, + CAST(false AS BOOLEAN) AS is_proxy_minimal, + uses_push0, CAST(NULL AS TIMESTAMP) AS decoded_at + FROM bytecode_metadata + "# + } + (false, true) => { + r#" + SELECT + contract_address, language, compiler_version, has_source_hash, + is_erc20, is_erc721, is_erc1155, is_proxy_eip1967, + is_proxy_minimal, uses_push0, decoded_at + FROM bytecode_metadata_v2 + "# + } + (true, true) => { + r#" + SELECT + contract_address, language, compiler_version, has_source_hash, + is_erc20, is_erc721, is_erc1155, is_proxy_eip1967, + is_proxy_minimal, uses_push0, decoded_at + FROM bytecode_metadata_v2 + UNION ALL + SELECT + v1.contract_address, v1.language, v1.compiler_version, + v1.has_source_hash, v1.is_erc20, v1.is_erc721, v1.is_erc1155, + v1.is_proxy_eip1967, CAST(false AS BOOLEAN) AS is_proxy_minimal, + v1.uses_push0, CAST(NULL AS TIMESTAMP) AS decoded_at + FROM bytecode_metadata v1 + WHERE NOT EXISTS ( + SELECT 1 + FROM bytecode_metadata_v2 v2 + WHERE v2.contract_address = v1.contract_address + ) + "# + } + }; + + conn.execute_batch(&format!( + r#" + CREATE OR REPLACE TEMP VIEW bytecode_metadata_current AS + {address_meta}; + "# + )) + .context("create combined metadata view") +} + +fn create_enrichment_current_view(conn: &Connection) -> Result<()> { + let sql = if table_exists(conn, "enrichment")? { + let chain_id = if column_exists(conn, "enrichment", "chain_id")? { + "chain_id" + } else { + "1::UBIGINT AS chain_id" + }; + let verification_source = if column_exists(conn, "enrichment", "verification_source")? { + "verification_source" + } else { + "CAST(NULL AS VARCHAR) AS verification_source" + }; + let match_type = if column_exists(conn, "enrichment", "match_type")? { + "match_type" + } else { + "CAST(NULL AS VARCHAR) AS match_type" + }; + let block_number = if column_exists(conn, "enrichment", "block_number")? { + "block_number" + } else { + "CAST(NULL AS UINTEGER) AS block_number" + }; + let create_index = if column_exists(conn, "enrichment", "create_index")? { + "create_index" + } else { + "CAST(NULL AS UINTEGER) AS create_index" + }; + format!( + r#" + CREATE OR REPLACE TEMP VIEW enrichment_current AS + SELECT + contract_address, + {chain_id}, + is_verified, + contract_name, + checked_at, + {verification_source}, + {match_type}, + {block_number}, + {create_index} + FROM enrichment; + "# + ) + } else { + r#" + CREATE OR REPLACE TEMP VIEW enrichment_current AS + SELECT + CAST(NULL AS BLOB) AS contract_address, + CAST(NULL AS UBIGINT) AS chain_id, + CAST(NULL AS BOOLEAN) AS is_verified, + CAST(NULL AS VARCHAR) AS contract_name, + CAST(NULL AS TIMESTAMP) AS checked_at, + CAST(NULL AS VARCHAR) AS verification_source, + CAST(NULL AS VARCHAR) AS match_type, + CAST(NULL AS UINTEGER) AS block_number, + CAST(NULL AS UINTEGER) AS create_index + WHERE FALSE; + "# + .to_string() + }; + conn.execute_batch(&sql) + .context("create enrichment compatibility view") +} + +/// Views consumed by `POST /api/query` users: `bytecodes`, +/// `decoded_bytecodes`, `contract_metadata_all`. +fn create_standard_query_views(conn: &Connection) -> Result<()> { + let has_rollups = table_exists(conn, "rollup_code_counts")?; + let has_zellic_bytecodes = table_exists(conn, "zellic_bytecodes")?; + let has_hash = table_exists(conn, "bytecode_metadata_by_hash")?; + let hash_has_decoded_at = + has_hash && column_exists(conn, "bytecode_metadata_by_hash", "decoded_at")?; + + // Distinct bytecodes with usage counts, from the native rollup instead of + // a parquet scan. `code` comes from the Zellic snapshot when available. + let bytecodes_sql = if has_rollups { + let (code_expr, code_join) = if has_zellic_bytecodes { + ( + "b.code", + "LEFT JOIN zellic_bytecodes b ON c.code_hash = b.code_hash", + ) + } else { + ("CAST(NULL AS BLOB)", "") + }; + format!( + r#" + CREATE OR REPLACE TEMP VIEW bytecodes AS + SELECT + c.code_hash, + lower('0x' || hex(c.code_hash)) AS code_hash_hex, + any_value(c.n_code_bytes)::UINTEGER AS n_code_bytes, + any_value({code_expr}) AS code, + SUM(c.contract_count)::UBIGINT AS contract_count + FROM rollup_code_counts c + {code_join} + GROUP BY c.code_hash; + "# + ) + } else { + r#" + CREATE OR REPLACE TEMP VIEW bytecodes AS + SELECT + code_hash, + lower('0x' || hex(code_hash)) AS code_hash_hex, + any_value(n_code_bytes)::UINTEGER AS n_code_bytes, + any_value(code) AS code, + COUNT(*)::UBIGINT AS contract_count + FROM contracts + WHERE code_hash IS NOT NULL + GROUP BY code_hash; + "# + .to_string() + }; + conn.execute_batch(&bytecodes_sql) + .context("create bytecodes query view")?; + + let decoded_sql = if has_hash { + let decoded_at = if hash_has_decoded_at { + "decoded_at" + } else { + "CAST(NULL AS TIMESTAMP) AS decoded_at" + }; + let decoded_order = if hash_has_decoded_at { + "decoded_at DESC NULLS LAST" + } else { + "code_hash" + }; + format!( + r#" + CREATE OR REPLACE TEMP VIEW decoded_bytecodes AS + SELECT + code_hash, + lower('0x' || hex(code_hash)) AS code_hash_hex, + language, + compiler_version, + has_source_hash, + is_erc20, + is_erc721, + is_erc1155, + is_proxy_eip1967, + is_proxy_minimal, + uses_push0, + decoded_at + FROM ( + SELECT + code_hash, + language, + compiler_version, + has_source_hash, + is_erc20, + is_erc721, + is_erc1155, + is_proxy_eip1967, + is_proxy_minimal, + uses_push0, + {decoded_at}, + row_number() OVER ( + PARTITION BY code_hash + ORDER BY {decoded_order} + ) AS rn + FROM bytecode_metadata_by_hash + ) + WHERE rn = 1; + "# + ) + } else { + r#" + CREATE OR REPLACE TEMP VIEW decoded_bytecodes AS + SELECT + CAST(NULL AS BLOB) AS code_hash, + CAST(NULL AS VARCHAR) AS code_hash_hex, + CAST(NULL AS VARCHAR) AS language, + CAST(NULL AS VARCHAR) AS compiler_version, + CAST(false AS BOOLEAN) AS has_source_hash, + CAST(false AS BOOLEAN) AS is_erc20, + CAST(false AS BOOLEAN) AS is_erc721, + CAST(false AS BOOLEAN) AS is_erc1155, + CAST(false AS BOOLEAN) AS is_proxy_eip1967, + CAST(false AS BOOLEAN) AS is_proxy_minimal, + CAST(false AS BOOLEAN) AS uses_push0, + CAST(NULL AS TIMESTAMP) AS decoded_at + WHERE FALSE; + "# + .to_string() + }; + conn.execute_batch(&decoded_sql) + .context("create decoded bytecodes query view")?; + + create_contract_metadata_view(conn) +} + +/// The SQL-explorer surface: `contract_metadata_all` / `contract_metadata`. +/// +/// Prefers the materialized `contract_metadata_native` table (see +/// `db::explorer`) with deployments newer than its bounds unioned in live and +/// undecorated; falls back to a live join over the native tables until the +/// materialization has been built. `transaction_hash` / `block_hash` / +/// `factory` are always NULL here — query the raw `contracts` view when +/// those are needed. +pub(crate) fn create_contract_metadata_view(conn: &Connection) -> Result<()> { + // Columns not carried by the native deployments table. + const NULL_RAW_COLS: &str = r#" + CAST(NULL AS BLOB) AS transaction_hash, + CAST(NULL AS VARCHAR) AS tx_hash, + CAST(NULL AS BLOB) AS block_hash, + CAST(NULL AS VARCHAR) AS block_hash_hex, + "#; + const NULL_FACTORY_COLS: &str = r#" + CAST(NULL AS BLOB) AS factory, + CAST(NULL AS VARCHAR) AS factory_address, + "#; + + let registry_loaded = table_exists(conn, "verification_registry_imports")?; + let has_materialized = table_exists(conn, "contract_metadata_native")? + && table_exists(conn, "contract_metadata_bounds")?; + let has_deployments = table_exists(conn, "contract_deployments_native")?; + + let body = if has_materialized { + let live_verified = if registry_loaded { + "CAST(false AS BOOLEAN)" + } else { + "CAST(NULL AS BOOLEAN)" + }; + format!( + r#" + SELECT + chain_id, + block_number, + create_index, + contract_address, + lower('0x' || hex(contract_address)) AS address, + {NULL_RAW_COLS} + deployer, + lower('0x' || hex(deployer)) AS deployer_address, + {NULL_FACTORY_COLS} + code_hash, + lower('0x' || hex(code_hash)) AS code_hash_hex, + n_code_bytes, + language, + compiler_version, + has_source_hash, + is_erc20, + is_erc721, + is_erc1155, + is_proxy_eip1967, + is_proxy_minimal, + uses_push0, + decoded_at, + is_verified, + contract_name, + verification_source, + match_type, + verification_checked_at + FROM contract_metadata_native + UNION ALL + SELECT + c.chain_id, + c.block_number, + c.create_index, + c.contract_address, + lower('0x' || hex(c.contract_address)) AS address, + {NULL_RAW_COLS} + c.deployer, + lower('0x' || hex(c.deployer)) AS deployer_address, + {NULL_FACTORY_COLS} + c.code_hash, + lower('0x' || hex(c.code_hash)) AS code_hash_hex, + c.n_code_bytes, + CAST(NULL AS VARCHAR) AS language, + CAST(NULL AS VARCHAR) AS compiler_version, + CAST(false AS BOOLEAN) AS has_source_hash, + CAST(false AS BOOLEAN) AS is_erc20, + CAST(false AS BOOLEAN) AS is_erc721, + CAST(false AS BOOLEAN) AS is_erc1155, + CAST(false AS BOOLEAN) AS is_proxy_eip1967, + CAST(false AS BOOLEAN) AS is_proxy_minimal, + CAST(false AS BOOLEAN) AS uses_push0, + CAST(NULL AS TIMESTAMP) AS decoded_at, + {live_verified} AS is_verified, + CAST(NULL AS VARCHAR) AS contract_name, + CAST(NULL AS VARCHAR) AS verification_source, + CAST(NULL AS VARCHAR) AS match_type, + CAST(NULL AS TIMESTAMP) AS verification_checked_at + FROM contract_deployments_native c + LEFT JOIN contract_metadata_bounds b ON c.chain_id = b.chain_id + WHERE b.chain_id IS NULL OR c.block_number > b.max_block + "# + ) + } else if has_deployments { + let is_verified_expr = if registry_loaded { + "COALESCE(e.is_verified, false) AS is_verified" + } else { + "e.is_verified" + }; + format!( + r#" + SELECT + c.chain_id, + c.block_number, + c.create_index, + c.contract_address, + lower('0x' || hex(c.contract_address)) AS address, + {NULL_RAW_COLS} + c.deployer, + lower('0x' || hex(c.deployer)) AS deployer_address, + {NULL_FACTORY_COLS} + c.code_hash, + lower('0x' || hex(c.code_hash)) AS code_hash_hex, + c.n_code_bytes, + m.language, + m.compiler_version, + COALESCE(m.has_source_hash, false) AS has_source_hash, + COALESCE(m.is_erc20, false) AS is_erc20, + COALESCE(m.is_erc721, false) AS is_erc721, + COALESCE(m.is_erc1155, false) AS is_erc1155, + COALESCE(m.is_proxy_eip1967, false) AS is_proxy_eip1967, + COALESCE(m.is_proxy_minimal, false) AS is_proxy_minimal, + COALESCE(m.uses_push0, false) AS uses_push0, + m.decoded_at, + {is_verified_expr}, + e.contract_name, + e.verification_source, + e.match_type, + e.checked_at AS verification_checked_at + FROM contract_deployments_native c + LEFT JOIN decoded_bytecodes m ON c.code_hash = m.code_hash + LEFT JOIN enrichment_current e + ON c.contract_address = e.contract_address + AND c.chain_id = e.chain_id + "# + ) + } else { + format!( + r#" + SELECT + CAST(NULL AS UBIGINT) AS chain_id, + CAST(NULL AS UINTEGER) AS block_number, + CAST(NULL AS UINTEGER) AS create_index, + CAST(NULL AS BLOB) AS contract_address, + CAST(NULL AS VARCHAR) AS address, + {NULL_RAW_COLS} + CAST(NULL AS BLOB) AS deployer, + CAST(NULL AS VARCHAR) AS deployer_address, + {NULL_FACTORY_COLS} + CAST(NULL AS BLOB) AS code_hash, + CAST(NULL AS VARCHAR) AS code_hash_hex, + CAST(NULL AS UINTEGER) AS n_code_bytes, + CAST(NULL AS VARCHAR) AS language, + CAST(NULL AS VARCHAR) AS compiler_version, + CAST(false AS BOOLEAN) AS has_source_hash, + CAST(false AS BOOLEAN) AS is_erc20, + CAST(false AS BOOLEAN) AS is_erc721, + CAST(false AS BOOLEAN) AS is_erc1155, + CAST(false AS BOOLEAN) AS is_proxy_eip1967, + CAST(false AS BOOLEAN) AS is_proxy_minimal, + CAST(false AS BOOLEAN) AS uses_push0, + CAST(NULL AS TIMESTAMP) AS decoded_at, + CAST(NULL AS BOOLEAN) AS is_verified, + CAST(NULL AS VARCHAR) AS contract_name, + CAST(NULL AS VARCHAR) AS verification_source, + CAST(NULL AS VARCHAR) AS match_type, + CAST(NULL AS TIMESTAMP) AS verification_checked_at + WHERE FALSE + "# + ) + }; + + conn.execute_batch(&format!( + r#" + CREATE OR REPLACE TEMP VIEW contract_metadata_all AS + {body}; + + CREATE OR REPLACE TEMP VIEW contract_metadata AS + SELECT * FROM contract_metadata_all; + "# + )) + .context("create contract metadata query view") +} diff --git a/src/decode/bytecode_meta.rs b/src/decode/bytecode_meta.rs index d6e3818..3bee14b 100644 --- a/src/decode/bytecode_meta.rs +++ b/src/decode/bytecode_meta.rs @@ -296,91 +296,3 @@ fn scan_opcodes(code: &[u8], meta: &mut BytecodeMetadata) { meta.is_erc20 = true; } } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn empty_bytecode() { - let m = analyze(&[]); - assert!(m.compiler_version.is_none()); - assert!(!m.is_erc20); - } - - #[test] - fn detects_push0() { - let code = [0x5f, 0x00]; - let m = analyze(&code); - assert!(m.uses_push0); - } - - #[test] - fn detects_erc20_selectors() { - let mut code = vec![]; - for sel in [ - [0x18u8, 0x16, 0x0d, 0xdd], - [0xa9, 0x05, 0x9c, 0xbb], - [0xdd, 0x62, 0xed, 0x3e], - ] { - code.push(0x63); - code.extend_from_slice(&sel); - } - let m = analyze(&code); - assert!(m.is_erc20); - } - - #[test] - fn parses_solc_metadata() { - // CBOR: a1 64 73 6f 6c 63 43 00 08 14 → { "solc": h'000814' } (= 0.8.20) - // length suffix: 00 0a (10 bytes) - let code = vec![ - 0xa1, 0x64, 0x73, 0x6f, 0x6c, 0x63, 0x43, 0x00, 0x08, 0x14, 0x00, 0x0a, - ]; - let m = analyze(&code); - assert_eq!(m.language, Some(Language::Solidity)); - assert_eq!(m.compiler_version.as_deref(), Some("0.8.20")); - } - - #[test] - fn detects_eip1167_minimal_proxy() { - // Construct the canonical 45-byte runtime with a dummy impl address. - let mut code = vec![]; - code.extend_from_slice(&[0x36, 0x3d, 0x3d, 0x37, 0x3d, 0x3d, 0x3d, 0x36, 0x3d, 0x73]); - code.extend_from_slice(&[0xab; 20]); // implementation address - code.extend_from_slice(&[ - 0x5a, 0xf4, 0x3d, 0x82, 0x80, 0x3e, 0x90, 0x3d, 0x91, 0x60, 0x2b, 0x57, 0xfd, 0x5b, - 0xf3, - ]); - assert_eq!(code.len(), 45); - let m = analyze(&code); - assert!(m.is_proxy_minimal, "should detect EIP-1167 minimal proxy"); - // Should not also flag as a different proxy type. - assert!(!m.is_proxy_eip1967); - } - - #[test] - fn rejects_minimal_proxy_with_wrong_length() { - // Same shape but one byte short — not a valid EIP-1167. - let mut code = vec![0x36, 0x3d, 0x3d, 0x37, 0x3d, 0x3d, 0x3d, 0x36, 0x3d, 0x73]; - code.extend_from_slice(&[0xab; 19]); - code.extend_from_slice(&[ - 0x5a, 0xf4, 0x3d, 0x82, 0x80, 0x3e, 0x90, 0x3d, 0x91, 0x60, 0x2b, 0x57, 0xfd, 0x5b, - 0xf3, - ]); - assert_eq!(code.len(), 44); - let m = analyze(&code); - assert!(!m.is_proxy_minimal); - } - - #[test] - fn ignores_absurd_cbor_map_count() { - let code = vec![ - 0xba, 0x99, 0xb6, 0x26, 0x57, // map(2578851415) - 0x00, 0x05, // metadata length: 5 bytes - ]; - let m = analyze(&code); - assert!(m.language.is_none()); - assert!(m.compiler_version.is_none()); - } -} diff --git a/src/decode/mod.rs b/src/decode/mod.rs index 1629bf9..1024ede 100644 --- a/src/decode/mod.rs +++ b/src/decode/mod.rs @@ -6,7 +6,7 @@ //! Resumable: re-running is a no-op for files already marked complete //! (unless `--overwrite`). //! -mod bytecode_meta; +pub mod bytecode_meta; use std::{ fs::File, diff --git a/src/extract/batch.rs b/src/extract/batch.rs index 270ae04..72c6fb6 100644 --- a/src/extract/batch.rs +++ b/src/extract/batch.rs @@ -169,7 +169,7 @@ fn is_rate_limited(err: &JsonRpcError) -> bool { err.code == 429 || err.message.to_ascii_lowercase().contains("compute units") } -fn decode_trace_block_value( +pub fn decode_trace_block_value( block: u64, value: serde_json::Value, ) -> Result> { @@ -197,32 +197,3 @@ fn is_reward_trace(item: &serde_json::Value) -> bool { .and_then(|action| action.get("rewardType")) .is_some() } - -#[cfg(test)] -mod tests { - use super::decode_trace_block_value; - - #[test] - fn trace_decode_ignores_gnosis_external_reward_traces() { - let value = serde_json::json!([ - { - "action": { - "author": "0x0000000000000000000000000000000000000000", - "rewardType": "external", - "value": "0x0" - }, - "blockHash": "0x0000000000000000000000000000000000000000000000000000000000000000", - "blockNumber": 46630628, - "result": null, - "subtraces": 0, - "traceAddress": [], - "transactionHash": null, - "transactionPosition": null, - "type": "reward" - } - ]); - - let traces = decode_trace_block_value(46630628, value).unwrap(); - assert!(traces.is_empty()); - } -} diff --git a/src/extract/mod.rs b/src/extract/mod.rs index 4a42c07..1dd3ac7 100644 --- a/src/extract/mod.rs +++ b/src/extract/mod.rs @@ -1,4 +1,4 @@ -mod batch; +pub mod batch; mod parquet_io; pub(crate) mod tail; mod traces; diff --git a/src/extract/tail.rs b/src/extract/tail.rs index 75f992b..f623e5a 100644 --- a/src/extract/tail.rs +++ b/src/extract/tail.rs @@ -13,7 +13,7 @@ use futures::{stream, StreamExt}; use super::{batch::BatchClient, parquet_io, traces::extract_contracts}; use crate::{db::Db, types::ChunkReport}; -const TAIL_BATCH_BLOCK_LIMIT: u64 = 5_000; +const TAIL_BATCH_BLOCK_LIMIT: u64 = 1_000; pub async fn rpc_chain_id(rpc_url: &str) -> Result { let provider = @@ -51,6 +51,12 @@ pub async fn tail_once( return Ok(None); } let end_block = (start_block + TAIL_BATCH_BLOCK_LIMIT - 1).min(target); + tracing::info!( + "tail scanning chain_id={} blocks {}-{}", + chain_id, + start_block, + end_block + ); let started_at = Utc::now(); let output_path = data_dir.join(format!( @@ -145,7 +151,7 @@ pub async fn tail_once( })?; } - db.refresh_contracts_view().await?; + db.refresh().await?; let size_bytes = std::fs::metadata(&output_path).ok().map(|m| m.len()); Ok(Some(ChunkReport { diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..ff4b776 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,10 @@ +pub mod blocks; +pub mod chains; +pub mod cli; +pub mod db; +pub mod decode; +pub mod extract; +pub mod load; +pub mod serve; +pub mod types; +pub mod util; diff --git a/src/load.rs b/src/load.rs index b255f21..563234a 100644 --- a/src/load.rs +++ b/src/load.rs @@ -2,7 +2,7 @@ use std::{ path::{Path, PathBuf}, - time::Instant, + time::{Instant, UNIX_EPOCH}, }; use anyhow::{anyhow, Context, Result}; @@ -104,6 +104,7 @@ fn run_load_blocking(args: LoadArgs) -> Result<()> { &args.memory_limit, args.threads, args.chain_id, + args.rebuild_va, )?; } @@ -111,21 +112,29 @@ fn run_load_blocking(args: LoadArgs) -> Result<()> { } #[derive(Debug)] -struct LoadInputs { - csv_contracts: PathBuf, - csv_bytecodes: PathBuf, - has_normalized_csv: bool, - parquet_files: Vec, +pub struct LoadInputs { + pub csv_contracts: PathBuf, + pub csv_bytecodes: PathBuf, + pub has_normalized_csv: bool, + pub parquet_files: Vec, } #[derive(Debug)] -struct VerifierAllianceInputs { - root: PathBuf, - contract_deployments: Vec, - verified_contracts: Vec, +pub struct VerifierAllianceInputs { + pub contract_deployments: Vec, + pub verified_contracts: Vec, } -fn detect_inputs(contracts_dir: &Path, contracts_glob: &str) -> Result { +#[derive(Debug, Clone)] +struct VaFileEntry { + table_name: &'static str, + path: PathBuf, + path_key: String, + size_bytes: i64, + modified_unix_ns: i64, +} + +pub fn detect_inputs(contracts_dir: &Path, contracts_glob: &str) -> Result { let csv_contracts = contracts_dir.join("contracts.csv"); let csv_bytecodes = contracts_dir.join("bytecodes.csv"); let has_normalized_csv = csv_contracts.is_file() && csv_bytecodes.is_file(); @@ -139,7 +148,9 @@ fn detect_inputs(contracts_dir: &Path, contracts_glob: &str) -> Result) -> Result> { +pub fn detect_verifier_alliance_inputs( + root: Option<&Path>, +) -> Result> { let Some(root) = root else { return Ok(None); }; @@ -167,13 +178,12 @@ fn detect_verifier_alliance_inputs(root: Option<&Path>) -> Result bool { } } -fn list_parquet_files(dir: &Path, glob: &str) -> Result> { +pub fn list_parquet_files(dir: &Path, glob: &str) -> Result> { let mut out: Vec = std::fs::read_dir(dir) .with_context(|| format!("read dir {}", dir.display()))? .filter_map(|e| e.ok()) @@ -305,6 +315,11 @@ fn load_normalized_csvs( } if overwrite { + // The snapshot is about to be rebuilt: subtract its rows from the + // deployment rollups so the next `blink serve` re-ingests the new + // data instead of trusting the stale `zellic://contracts` source. + crate::db::invalidate_zellic_rollups(&conn) + .context("invalidate zellic deployment rollups")?; conn.execute_batch( r#" DROP TABLE IF EXISTS zellic_bytecode_counts; @@ -427,6 +442,7 @@ fn ensure_verification_schema(conn: &Connection) -> Result<()> { ALTER TABLE enrichment ADD COLUMN IF NOT EXISTS create_index UINTEGER; UPDATE enrichment SET chain_id = 1 WHERE chain_id IS NULL; CREATE INDEX IF NOT EXISTS enrichment_chain_addr_idx ON enrichment(chain_id, contract_address); + CREATE INDEX IF NOT EXISTS enrichment_chain_block_idx ON enrichment(chain_id, block_number); CREATE INDEX IF NOT EXISTS enrichment_verified_idx ON enrichment(is_verified); CREATE INDEX IF NOT EXISTS enrichment_source_idx ON enrichment(verification_source); @@ -437,6 +453,30 @@ fn ensure_verification_schema(conn: &Connection) -> Result<()> { verified_count UBIGINT NOT NULL, PRIMARY KEY (source, chain_id) ); + + CREATE TABLE IF NOT EXISTS verification_registry_files ( + source VARCHAR NOT NULL, + chain_id UBIGINT NOT NULL, + table_name VARCHAR NOT NULL, + path VARCHAR NOT NULL, + size_bytes BIGINT NOT NULL, + modified_unix_ns BIGINT NOT NULL, + imported_at TIMESTAMP NOT NULL + ); + CREATE UNIQUE INDEX IF NOT EXISTS verification_registry_files_idx + ON verification_registry_files(source, chain_id, table_name, path); + + CREATE TABLE IF NOT EXISTS verification_registry_file_addresses ( + source VARCHAR NOT NULL, + chain_id UBIGINT NOT NULL, + table_name VARCHAR NOT NULL, + path VARCHAR NOT NULL, + contract_address BLOB NOT NULL + ); + CREATE INDEX IF NOT EXISTS verification_registry_file_addresses_idx + ON verification_registry_file_addresses(source, chain_id, table_name, path); + CREATE INDEX IF NOT EXISTS verification_registry_file_addresses_addr_idx + ON verification_registry_file_addresses(chain_id, contract_address); "#, ) .context("create verification registry schema") @@ -448,9 +488,17 @@ fn load_verifier_alliance_registry( memory_limit: &str, threads: Option, chain_id: u64, + rebuild_va: bool, ) -> Result<()> { let started = Instant::now(); - print_kv("step", "import Verifier Alliance registry"); + print_kv( + "step", + if rebuild_va { + "rebuild Verifier Alliance registry" + } else { + "import Verifier Alliance registry incrementally" + }, + ); let db_path = data_dir.join("blink.duckdb"); let conn = @@ -458,16 +506,29 @@ fn load_verifier_alliance_registry( configure_duckdb(&conn, memory_limit, threads)?; ensure_verification_schema(&conn)?; - let deployments_glob = sql_path(&inputs.root.join("contract_deployments").join("*.parquet")); - let verifications_glob = sql_path(&inputs.root.join("verified_contracts").join("*.parquet")); + let entries = verifier_alliance_file_entries(inputs)?; + if rebuild_va { + rebuild_verifier_alliance_registry(&conn, inputs, &entries, chain_id, started) + } else { + import_verifier_alliance_registry_incremental(&conn, inputs, &entries, chain_id, started) + } +} +fn rebuild_verifier_alliance_registry( + conn: &Connection, + inputs: &VerifierAllianceInputs, + entries: &[VaFileEntry], + chain_id: u64, + started: Instant, +) -> Result<()> { + let deployments_list = sql_path_list(&inputs.contract_deployments)?; + let verifications_list = sql_path_list(&inputs.verified_contracts)?; conn.execute_batch("BEGIN TRANSACTION;") .context("begin Verifier Alliance import")?; let result = (|| -> Result<()> { conn.execute_batch(&format!( r#" - DROP TABLE IF EXISTS enrichment_next; DELETE FROM verification_registry_imports WHERE source = 'verifier_alliance' AND chain_id = {chain_id}; @@ -492,14 +553,14 @@ fn load_verifier_alliance_registry( bool_or(COALESCE(vc.creation_match, false)) AS creation_match, bool_or(COALESCE(vc.runtime_metadata_match, false)) AS runtime_metadata_match, bool_or(COALESCE(vc.creation_metadata_match, false)) AS creation_metadata_match - FROM read_parquet('{verifications_glob}') vc - JOIN read_parquet('{deployments_glob}') cd + FROM read_parquet({verifications_list}) vc + JOIN read_parquet({deployments_list}) cd ON cd.id = vc.deployment_id WHERE cd.chain_id = {chain_id} AND cd.address IS NOT NULL GROUP BY cd.address; - CREATE TABLE enrichment_next AS + CREATE OR REPLACE TEMP TABLE enrichment_next AS SELECT contract_address, {chain_id}::UBIGINT AS chain_id, @@ -557,6 +618,8 @@ fn load_verifier_alliance_registry( ); conn.execute_batch(&sql) .context("import Verifier Alliance verified addresses")?; + rebuild_va_file_addresses(conn, inputs, chain_id)?; + replace_va_file_manifest(conn, chain_id, entries)?; Ok(()) })(); @@ -567,24 +630,417 @@ fn load_verifier_alliance_registry( conn.execute_batch("COMMIT;") .context("commit Verifier Alliance import")?; - let verified: i64 = conn - .query_row( - "SELECT COUNT(*) FROM enrichment WHERE verification_source = 'verifier_alliance' AND chain_id = ?", - params![chain_id], - |row| row.get(0), - ) - .unwrap_or(0); + let verified = va_verified_count(conn, chain_id)?; print_kv_accent( "verified", &format!( "{} from Verifier Alliance · {:.1}s", - format_count(verified.max(0) as u64), + format_count(verified), started.elapsed().as_secs_f64() ), ); Ok(()) } +fn import_verifier_alliance_registry_incremental( + conn: &Connection, + inputs: &VerifierAllianceInputs, + entries: &[VaFileEntry], + chain_id: u64, + started: Instant, +) -> Result<()> { + let changed_entries = changed_va_file_entries(conn, chain_id, entries)?; + let import_exists = verification_registry_import_exists(conn, chain_id)?; + let deployment_changed = changed_entries + .iter() + .any(|entry| entry.table_name == "contract_deployments"); + + let mut changed_verified = if deployment_changed || !import_exists { + entries + .iter() + .filter(|entry| entry.table_name == "verified_contracts") + .cloned() + .collect::>() + } else { + changed_entries + .iter() + .filter(|entry| entry.table_name == "verified_contracts") + .cloned() + .collect::>() + }; + changed_verified.sort_by_key(|entry| entry.path_key.clone()); + changed_verified.dedup_by(|a, b| a.path_key == b.path_key); + + if changed_entries.is_empty() && import_exists { + let verified = va_verified_count(conn, chain_id)?; + print_kv_accent( + "verified", + &format!( + "{} from Verifier Alliance · already current · {:.1}s", + format_count(verified), + started.elapsed().as_secs_f64() + ), + ); + return Ok(()); + } + + if changed_verified.is_empty() { + upsert_va_file_manifest_entries(conn, chain_id, &changed_entries)?; + let verified = va_verified_count(conn, chain_id)?; + print_kv_accent( + "verified", + &format!( + "{} from Verifier Alliance · metadata refreshed · {:.1}s", + format_count(verified), + started.elapsed().as_secs_f64() + ), + ); + return Ok(()); + } + + let deployments_list = sql_path_list(&inputs.contract_deployments)?; + let verifications_list = sql_path_list(&inputs.verified_contracts)?; + let changed_verifications_list = + sql_path_list(changed_verified.iter().map(|entry| &entry.path))?; + let changed_file_values = + sql_string_values(changed_verified.iter().map(|entry| entry.path_key.as_str())); + + conn.execute_batch("BEGIN TRANSACTION;") + .context("begin incremental Verifier Alliance import")?; + + let result = (|| -> Result<()> { + let sql = format!( + r#" + CREATE OR REPLACE TEMP TABLE va_changed_files AS + SELECT col0 AS path + FROM (VALUES {changed_file_values}); + + CREATE OR REPLACE TEMP TABLE va_previous_affected_addresses AS + SELECT DISTINCT contract_address + FROM verification_registry_file_addresses + WHERE source = 'verifier_alliance' + AND chain_id = {chain_id} + AND table_name = 'verified_contracts' + AND path IN (SELECT path FROM va_changed_files); + + CREATE OR REPLACE TEMP TABLE va_changed_file_addresses AS + SELECT DISTINCT + vc.filename AS path, + cd.address AS contract_address + FROM read_parquet({changed_verifications_list}, filename=true) vc + JOIN read_parquet({deployments_list}) cd + ON cd.id = vc.deployment_id + WHERE cd.chain_id = {chain_id} + AND cd.address IS NOT NULL; + + CREATE OR REPLACE TEMP TABLE va_affected_addresses AS + SELECT contract_address FROM va_previous_affected_addresses + UNION + SELECT contract_address FROM va_changed_file_addresses; + + DELETE FROM enrichment + WHERE verification_source = 'verifier_alliance' + AND chain_id = {chain_id} + AND contract_address IN ( + SELECT contract_address FROM va_affected_addresses + ); + + DELETE FROM verification_registry_file_addresses + WHERE source = 'verifier_alliance' + AND chain_id = {chain_id} + AND table_name = 'verified_contracts' + AND path IN (SELECT path FROM va_changed_files); + + INSERT INTO verification_registry_file_addresses ( + source, + chain_id, + table_name, + path, + contract_address + ) + SELECT + 'verifier_alliance', + {chain_id}, + 'verified_contracts', + path, + contract_address + FROM va_changed_file_addresses; + + CREATE OR REPLACE TEMP TABLE va_verified_contracts AS + SELECT + cd.address AS contract_address, + (max(cd.block_number) FILTER (WHERE cd.block_number >= 0))::UINTEGER + AS block_number, + (min(cd.transaction_index) FILTER (WHERE cd.transaction_index >= 0))::UINTEGER + AS create_index, + max(vc.created_at)::TIMESTAMP AS checked_at, + bool_or(COALESCE(vc.runtime_match, false)) AS runtime_match, + bool_or(COALESCE(vc.creation_match, false)) AS creation_match, + bool_or(COALESCE(vc.runtime_metadata_match, false)) AS runtime_metadata_match, + bool_or(COALESCE(vc.creation_metadata_match, false)) AS creation_metadata_match + FROM read_parquet({verifications_list}) vc + JOIN read_parquet({deployments_list}) cd + ON cd.id = vc.deployment_id + JOIN va_affected_addresses affected + ON affected.contract_address = cd.address + WHERE cd.chain_id = {chain_id} + AND cd.address IS NOT NULL + GROUP BY cd.address; + + INSERT INTO enrichment ( + contract_address, + chain_id, + is_verified, + contract_name, + checked_at, + verification_source, + match_type, + block_number, + create_index + ) + SELECT + contract_address, + {chain_id}::UBIGINT AS chain_id, + true AS is_verified, + CAST(NULL AS VARCHAR) AS contract_name, + COALESCE(checked_at, CURRENT_TIMESTAMP) AS checked_at, + 'verifier_alliance' AS verification_source, + CASE + WHEN runtime_match AND creation_match THEN 'runtime+creation' + WHEN runtime_match THEN 'runtime' + WHEN creation_match THEN 'creation' + WHEN runtime_metadata_match OR creation_metadata_match THEN 'metadata' + ELSE 'verified' + END AS match_type, + block_number, + create_index + FROM va_verified_contracts; + + DELETE FROM verification_registry_imports + WHERE source = 'verifier_alliance' + AND chain_id = {chain_id}; + + INSERT INTO verification_registry_imports ( + source, + chain_id, + imported_at, + verified_count + ) + SELECT + 'verifier_alliance', + {chain_id}, + CURRENT_TIMESTAMP, + COUNT(*)::UBIGINT + FROM enrichment + WHERE verification_source = 'verifier_alliance' + AND chain_id = {chain_id}; + "# + ); + conn.execute_batch(&sql) + .context("import changed Verifier Alliance verified addresses")?; + upsert_va_file_manifest_entries(conn, chain_id, &changed_entries)?; + Ok(()) + })(); + + if let Err(err) = result { + let _ = conn.execute_batch("ROLLBACK;"); + return Err(err); + } + conn.execute_batch("COMMIT;") + .context("commit incremental Verifier Alliance import")?; + + let verified = va_verified_count(conn, chain_id)?; + print_kv_accent( + "verified", + &format!( + "{} from Verifier Alliance · {} changed file(s) · {:.1}s", + format_count(verified), + changed_entries.len(), + started.elapsed().as_secs_f64() + ), + ); + Ok(()) +} + +fn rebuild_va_file_addresses( + conn: &Connection, + inputs: &VerifierAllianceInputs, + chain_id: u64, +) -> Result<()> { + let deployments_list = sql_path_list(&inputs.contract_deployments)?; + let verifications_list = sql_path_list(&inputs.verified_contracts)?; + let sql = format!( + r#" + DELETE FROM verification_registry_file_addresses + WHERE source = 'verifier_alliance' + AND chain_id = {chain_id}; + + INSERT INTO verification_registry_file_addresses ( + source, + chain_id, + table_name, + path, + contract_address + ) + SELECT DISTINCT + 'verifier_alliance', + {chain_id}, + 'verified_contracts', + vc.filename, + cd.address + FROM read_parquet({verifications_list}, filename=true) vc + JOIN read_parquet({deployments_list}) cd + ON cd.id = vc.deployment_id + WHERE cd.chain_id = {chain_id} + AND cd.address IS NOT NULL; + "# + ); + conn.execute_batch(&sql) + .context("rebuild Verifier Alliance file address map") +} + +fn verification_registry_import_exists(conn: &Connection, chain_id: u64) -> Result { + let count: i64 = conn.query_row( + "SELECT COUNT(*) FROM verification_registry_imports WHERE source = 'verifier_alliance' AND chain_id = ?", + params![chain_id], + |row| row.get(0), + )?; + Ok(count > 0) +} + +fn va_verified_count(conn: &Connection, chain_id: u64) -> Result { + let count: i64 = conn.query_row( + "SELECT COUNT(*) FROM enrichment WHERE verification_source = 'verifier_alliance' AND chain_id = ?", + params![chain_id], + |row| row.get::<_, i64>(0), + )?; + Ok(count.max(0) as u64) +} + +fn verifier_alliance_file_entries(inputs: &VerifierAllianceInputs) -> Result> { + let mut entries = Vec::new(); + for path in &inputs.contract_deployments { + entries.push(va_file_entry("contract_deployments", path)?); + } + for path in &inputs.verified_contracts { + entries.push(va_file_entry("verified_contracts", path)?); + } + entries.sort_by_key(|entry| (entry.table_name, entry.path_key.clone())); + Ok(entries) +} + +fn va_file_entry(table_name: &'static str, path: &Path) -> Result { + let canonical = + std::fs::canonicalize(path).with_context(|| format!("canonicalize {}", path.display()))?; + let metadata = + std::fs::metadata(&canonical).with_context(|| format!("stat {}", canonical.display()))?; + let modified = metadata + .modified() + .ok() + .and_then(|time| time.duration_since(UNIX_EPOCH).ok()) + .map(|duration| { + duration.as_secs() as i64 * 1_000_000_000 + i64::from(duration.subsec_nanos()) + }) + .unwrap_or(0); + Ok(VaFileEntry { + table_name, + path_key: canonical.display().to_string(), + path: canonical, + size_bytes: metadata.len().min(i64::MAX as u64) as i64, + modified_unix_ns: modified, + }) +} + +fn changed_va_file_entries( + conn: &Connection, + chain_id: u64, + entries: &[VaFileEntry], +) -> Result> { + let mut changed = Vec::new(); + for entry in entries { + let count: i64 = conn.query_row( + r#" + SELECT COUNT(*) + FROM verification_registry_files + WHERE source = 'verifier_alliance' + AND chain_id = ? + AND table_name = ? + AND path = ? + AND size_bytes = ? + AND modified_unix_ns = ? + "#, + params![ + chain_id, + entry.table_name, + entry.path_key, + entry.size_bytes, + entry.modified_unix_ns + ], + |row| row.get(0), + )?; + if count == 0 { + changed.push(entry.clone()); + } + } + Ok(changed) +} + +fn replace_va_file_manifest( + conn: &Connection, + chain_id: u64, + entries: &[VaFileEntry], +) -> Result<()> { + conn.execute( + "DELETE FROM verification_registry_files WHERE source = 'verifier_alliance' AND chain_id = ?", + params![chain_id], + ) + .context("clear Verifier Alliance file manifest")?; + upsert_va_file_manifest_entries(conn, chain_id, entries) +} + +fn upsert_va_file_manifest_entries( + conn: &Connection, + chain_id: u64, + entries: &[VaFileEntry], +) -> Result<()> { + for entry in entries { + conn.execute( + r#" + DELETE FROM verification_registry_files + WHERE source = 'verifier_alliance' + AND chain_id = ? + AND table_name = ? + AND path = ? + "#, + params![chain_id, entry.table_name, entry.path_key], + ) + .context("delete Verifier Alliance file manifest row")?; + conn.execute( + r#" + INSERT INTO verification_registry_files ( + source, + chain_id, + table_name, + path, + size_bytes, + modified_unix_ns, + imported_at + ) + VALUES ('verifier_alliance', ?, ?, ?, ?, ?, CURRENT_TIMESTAMP) + "#, + params![ + chain_id, + entry.table_name, + entry.path_key, + entry.size_bytes, + entry.modified_unix_ns + ], + ) + .context("insert Verifier Alliance file manifest row")?; + } + Ok(()) +} + fn import_bytecodes(conn: &Connection, bytecodes_csv: &Path) -> Result<()> { let started = Instant::now(); print_kv("step", "import unique bytecodes"); @@ -714,149 +1170,32 @@ fn sql_path(path: &Path) -> String { path.display().to_string().replace('\'', "''") } -#[cfg(test)] -mod tests { - use super::*; - use std::{ - fs, - path::PathBuf, - time::{SystemTime, UNIX_EPOCH}, - }; - - struct TestDir { - path: PathBuf, - } - - impl TestDir { - fn new(name: &str) -> Self { - let unique = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_nanos(); - let path = std::env::temp_dir().join(format!( - "blink_load_test_{}_{}_{}", - std::process::id(), - name, - unique - )); - fs::create_dir_all(&path).unwrap(); - Self { path } - } - - fn touch(&self, name: &str) -> PathBuf { - let path = self.path.join(name); - fs::write(&path, []).unwrap(); - path - } +fn sql_path_list(paths: I) -> Result +where + I: IntoIterator, + P: AsRef, +{ + let mut out = Vec::new(); + for path in paths { + let path = path.as_ref(); + let canonical = std::fs::canonicalize(path) + .with_context(|| format!("canonicalize {}", path.display()))?; + out.push(format!("'{}'", sql_path(&canonical))); } - - impl Drop for TestDir { - fn drop(&mut self) { - let _ = fs::remove_dir_all(&self.path); - } + if out.is_empty() { + return Err(anyhow!("empty parquet path list")); } + Ok(format!("[{}]", out.join(", "))) +} - fn names(paths: &[PathBuf]) -> Vec { - paths - .iter() - .map(|path| { - path.file_name() - .and_then(|name| name.to_str()) - .unwrap() - .to_string() - }) - .collect() - } - - #[test] - fn detect_inputs_requires_both_csv_files_for_csv_load() { - let dir = TestDir::new("csv_pair"); - dir.touch("contracts.csv"); - - let inputs = detect_inputs(&dir.path, "*.parquet").unwrap(); - - assert!(!inputs.has_normalized_csv); - assert!(inputs.parquet_files.is_empty()); - - dir.touch("bytecodes.csv"); - let inputs = detect_inputs(&dir.path, "*.parquet").unwrap(); - - assert!(inputs.has_normalized_csv); - assert_eq!(inputs.csv_contracts, dir.path.join("contracts.csv")); - assert_eq!(inputs.csv_bytecodes, dir.path.join("bytecodes.csv")); - } - - #[test] - fn list_parquet_files_filters_hidden_and_non_matching_files() { - let dir = TestDir::new("parquet_filter"); - dir.touch("b.parquet"); - dir.touch("a.parquet"); - dir.touch(".hidden.parquet"); - dir.touch("contracts.csv"); - dir.touch("notes.txt"); - - let files = list_parquet_files(&dir.path, "*.parquet").unwrap(); - - assert_eq!(names(&files), vec!["a.parquet", "b.parquet"]); - } - - #[test] - fn detect_inputs_applies_parquet_glob() { - let dir = TestDir::new("parquet_glob"); - dir.touch("ethereum__contracts__1_to_2.parquet"); - dir.touch("other__contracts__1_to_2.parquet"); - - let inputs = detect_inputs(&dir.path, "ethereum__*.parquet").unwrap(); - - assert_eq!( - names(&inputs.parquet_files), - vec!["ethereum__contracts__1_to_2.parquet"] - ); - } - - #[test] - fn detect_verifier_alliance_inputs_requires_both_tables() { - let dir = TestDir::new("va_missing_table"); - fs::create_dir_all(dir.path.join("contract_deployments")).unwrap(); - - let err = detect_verifier_alliance_inputs(Some(&dir.path)).unwrap_err(); - - assert!(err.to_string().contains("--va needs both")); - } - - #[test] - fn detect_verifier_alliance_inputs_finds_required_parquet_files() { - let dir = TestDir::new("va_tables"); - let deployments = dir.path.join("contract_deployments"); - let verifications = dir.path.join("verified_contracts"); - fs::create_dir_all(&deployments).unwrap(); - fs::create_dir_all(&verifications).unwrap(); - fs::write(deployments.join("contract_deployments_0_1.parquet"), []).unwrap(); - fs::write(verifications.join("verified_contracts_0_1.parquet"), []).unwrap(); - - let inputs = detect_verifier_alliance_inputs(Some(&dir.path)) - .unwrap() - .unwrap(); - - assert_eq!(inputs.root, dir.path); - assert_eq!(inputs.contract_deployments.len(), 1); - assert_eq!(inputs.verified_contracts.len(), 1); - } - - #[test] - fn load_parquet_links_creates_symlinks_without_copying() { - let src = TestDir::new("link_src"); - let dst = TestDir::new("link_dst"); - let parquet = src.touch("contracts__0000000001__0000000002.parquet"); - - load_parquet_links(&src.path, &dst.path, std::slice::from_ref(&parquet), false).unwrap(); - - let linked = dst.path.join("contracts__0000000001__0000000002.parquet"); - let metadata = fs::symlink_metadata(&linked).unwrap(); - assert!(metadata.file_type().is_symlink()); - assert_eq!( - fs::canonicalize(linked).unwrap(), - fs::canonicalize(parquet).unwrap() - ); - } +fn sql_string_values(values: I) -> String +where + I: IntoIterator, + S: AsRef, +{ + values + .into_iter() + .map(|value| format!("('{}')", value.as_ref().replace('\'', "''"))) + .collect::>() + .join(", ") } diff --git a/src/main.rs b/src/main.rs index 3c9a346..efcc2b2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,20 +1,9 @@ //! `blink` binary entry point. -mod blocks; -mod chains; -mod cli; -mod db; -mod decode; -mod extract; -mod load; -mod serve; -mod types; -mod util; - use anyhow::Result; use tracing_subscriber::{fmt::time::ChronoLocal, EnvFilter}; -use crate::extract::run_contracts; +use blink::{cli, decode, extract::run_contracts, load, serve}; #[tokio::main] async fn main() -> Result<()> { diff --git a/src/serve.rs b/src/serve.rs index fa19757..35814d9 100644 --- a/src/serve.rs +++ b/src/serve.rs @@ -5,6 +5,12 @@ //! - repeated `--rpc URL` flags poll one or more chain heads and extract //! newly produced blocks into separate `tail__chain_*` parquet files. //! +//! Serving model: every cacheable endpoint is stale-while-revalidate. A +//! cached entry is returned immediately no matter its age; entries past +//! their TTL trigger a background refresh (deduplicated per key). Combined +//! with the rollup tables in `db`, cold misses are native-table queries, so +//! the server starts serving instantly and warms itself in the background. +//! //! Endpoints (all return JSON): //! - `GET /api/stats` — totals, verified pct, last block, verification coverage. //! - `GET /api/runtime` — serve-mode flags and background loop health. @@ -20,21 +26,30 @@ use std::{ future::Future, hash::Hash, net::SocketAddr, - sync::Arc, + path::PathBuf, + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, Mutex as StdMutex, + }, time::{Duration, Instant}, }; use anyhow::{Context, Result}; use axum::{ - extract::{Query, State}, + extract::{Query, Request, State}, http::{HeaderValue, Method, StatusCode}, - response::{IntoResponse, Json}, + middleware::{self, Next}, + response::{IntoResponse, Json, Response}, routing::get, }; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; -use tokio::{net::TcpListener, sync::Mutex}; +use tokio::{ + net::TcpListener, + sync::{watch, Mutex}, +}; use tower_http::{ + compression::CompressionLayer, cors::{Any, CorsLayer}, trace::TraceLayer, }; @@ -46,7 +61,7 @@ use crate::{ blocks::blocks_per_day, chains::{self, ChainInfo}, cli::ServeArgs, - db::{Db, RecentCursor}, + db::{Db, DbOptions, RecentCursor}, }; #[derive(Clone)] @@ -54,14 +69,117 @@ struct AppState { db: Db, runtime: Arc, cache: Arc, + latency: Arc, +} + +/// Upper bounds (ms) of the API latency histogram buckets; the last bucket is +/// open-ended. +const LATENCY_BUCKET_UPPER_MS: [u64; 11] = [1, 2, 5, 10, 25, 50, 100, 250, 500, 1000, 5000]; +const SLOW_REQUEST_LOG_THRESHOLD: Duration = Duration::from_millis(1000); + +/// Lock-free rolling latency histogram over every `/api/*` response since +/// startup. Answers "are we actually fast in production?" via `/api/runtime`. +#[derive(Default)] +struct LatencyStats { + buckets: [AtomicU64; LATENCY_BUCKET_UPPER_MS.len() + 1], + requests: AtomicU64, + total_micros: AtomicU64, } -const API_CACHE_TTL: Duration = Duration::from_secs(30); +#[derive(Debug, Default, Clone, Serialize, ToSchema)] +struct LatencySnapshot { + requests: u64, + avg_ms: f64, + p50_ms: f64, + p95_ms: f64, + p99_ms: f64, +} + +impl LatencyStats { + fn record(&self, elapsed: Duration) { + let ms = elapsed.as_millis() as u64; + let idx = LATENCY_BUCKET_UPPER_MS + .iter() + .position(|upper| ms <= *upper) + .unwrap_or(LATENCY_BUCKET_UPPER_MS.len()); + self.buckets[idx].fetch_add(1, Ordering::Relaxed); + self.requests.fetch_add(1, Ordering::Relaxed); + self.total_micros + .fetch_add(elapsed.as_micros() as u64, Ordering::Relaxed); + } + + fn percentile(&self, counts: &[u64], total: u64, q: f64) -> f64 { + if total == 0 { + return 0.0; + } + let target = (total as f64 * q).ceil() as u64; + let mut cumulative = 0u64; + for (idx, count) in counts.iter().enumerate() { + cumulative += count; + if cumulative >= target { + return LATENCY_BUCKET_UPPER_MS + .get(idx) + .copied() + .unwrap_or(LATENCY_BUCKET_UPPER_MS[LATENCY_BUCKET_UPPER_MS.len() - 1] * 2) + as f64; + } + } + 0.0 + } + + fn snapshot(&self) -> LatencySnapshot { + let counts: Vec = self + .buckets + .iter() + .map(|bucket| bucket.load(Ordering::Relaxed)) + .collect(); + let requests = self.requests.load(Ordering::Relaxed); + let avg_ms = if requests == 0 { + 0.0 + } else { + self.total_micros.load(Ordering::Relaxed) as f64 / requests as f64 / 1000.0 + }; + LatencySnapshot { + requests, + avg_ms, + p50_ms: self.percentile(&counts, requests, 0.50), + p95_ms: self.percentile(&counts, requests, 0.95), + p99_ms: self.percentile(&counts, requests, 0.99), + } + } +} + +/// Record latency for every API request; anything past the slow threshold is +/// logged so regressions surface in the journal, not just in percentiles. +async fn track_latency(State(state): State, req: Request, next: Next) -> Response { + let path_is_api = req.uri().path().starts_with("/api/"); + let method = req.method().clone(); + let path = req.uri().path().to_string(); + let started = Instant::now(); + let response = next.run(req).await; + if path_is_api { + let elapsed = started.elapsed(); + state.latency.record(elapsed); + if elapsed >= SLOW_REQUEST_LOG_THRESHOLD { + tracing::warn!( + "slow dashboard request: {} {} took {:.1}ms", + method, + path, + elapsed.as_secs_f64() * 1000.0 + ); + } + } + response +} + +const API_CACHE_TTL: Duration = Duration::from_secs(600); +const HIGHEST_BLOCK_TTL: Duration = Duration::from_secs(5); +const TAIL_START_DELAY: Duration = Duration::from_secs(15); const DEFAULT_COMPILER_LIMIT: u32 = 12; const DEFAULT_RECENT_LIMIT: u32 = 20; const INITIAL_DEPLOYS_RANGE: &str = "day"; const INITIAL_VERIFIED_RANGE: &str = "week"; -const PRESET_CHART_RANGES: &[&str] = &["hour", "day", "week", "month", "year"]; +const INITIAL_AGGREGATE_RANGE: &str = "month"; const API_TAG: &str = "Dashboard"; #[derive(OpenApi)] @@ -75,25 +193,37 @@ struct ApiCache { stats: CacheMap, deploys: CacheMap>, verified: CacheMap>, - bytecode_sizes: CacheMap>, - compilers: CacheMap, u64)>, + bytecode_sizes: CacheMap>, + compilers: CacheMap, u64)>, recent: CacheMap, languages: CacheMap>, - standards: CacheMap, + standards: CacheMap, + highest_blocks: CacheMap, } -#[derive(Clone, Copy, Eq, Hash, PartialEq)] -struct BucketCacheKey { +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub struct BucketCacheKey { chain_id: u64, bucket: u64, start_block: Option, end_block: Option, } -#[derive(Clone, Copy, Eq, Hash, PartialEq)] -struct LimitCacheKey { +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub struct RangeCacheKey { + chain_id: u64, + bucket: Option, + start_block: Option, + end_block: Option, +} + +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +struct LimitRangeCacheKey { chain_id: u64, limit: u32, + bucket: Option, + start_block: Option, + end_block: Option, } #[derive(Clone, Copy, Eq, Hash, PartialEq)] @@ -104,60 +234,137 @@ struct RecentCacheKey { before_create_index: Option, } +/// Stale-while-revalidate, single-flight cache. +/// +/// Reads never block on the database once a key has been populated: expired +/// entries are served as-is while one background task per key refreshes them. +/// Cold misses are single-flight too — concurrent requests for the same key +/// (e.g. a dashboard fanning out while the boot prewarm runs) wait for the +/// one in-flight computation instead of duplicating it on a small host. struct CacheMap { - values: Mutex>, + inner: Arc>, +} + +struct CacheMapInner { + values: StdMutex>, + /// Keys currently being computed; waiters subscribe to the receiver and + /// wake when the compute holder drops its sender. + inflight: StdMutex>>, +} + +impl Clone for CacheMap { + fn clone(&self) -> Self { + Self { + inner: self.inner.clone(), + } + } } impl Default for CacheMap { fn default() -> Self { Self { - values: Mutex::new(HashMap::new()), + inner: Arc::new(CacheMapInner { + values: StdMutex::new(HashMap::new()), + inflight: StdMutex::new(HashMap::new()), + }), } } } impl CacheMap where - K: Copy + Eq + Hash, - V: Clone, + K: Copy + Eq + Hash + Send + Sync + 'static, + V: Clone + Send + Sync + 'static, { - async fn get_or_try_update(&self, key: K, ttl: Duration, future: F) -> Result + async fn get_or_refresh(&self, key: K, ttl: Duration, make: F) -> Result where - F: Future>, + F: FnOnce() -> Fut, + Fut: Future> + Send + 'static, { - { - let guard = self.values.lock().await; - if let Some((stored_at, value)) = guard.get(&key) { - if stored_at.elapsed() > ttl { - tracing::debug!("serving stale dashboard cache entry"); + let mut make = Some(make); + loop { + if let Some((age, value)) = self.lookup(&key) { + if age > ttl { + if let Ok(slot) = self.claim(key) { + let this = self.clone(); + let refresh = (make.take().expect("make consumed once"))(); + tokio::spawn(async move { + match refresh.await { + Ok(fresh) => this.insert(key, fresh), + Err(err) => tracing::warn!( + "background dashboard cache refresh failed: {:#}", + err + ), + } + this.release(&key); + drop(slot); + }); + } + } + return Ok(value); + } + + match self.claim(key) { + Ok(slot) => { + let result = (make.take().expect("make consumed once"))().await; + if let Ok(value) = &result { + self.insert(key, value.clone()); + } + self.release(&key); + drop(slot); + return result; + } + Err(mut waiter) => { + let _ = waiter.changed().await; } - return Ok(value.clone()); } } + } - let value = future.await?; - self.insert(key, value.clone()).await; - Ok(value) + fn lookup(&self, key: &K) -> Option<(Duration, V)> { + self.inner + .values + .lock() + .expect("cache map poisoned") + .get(key) + .map(|(at, value)| (at.elapsed(), value.clone())) } - async fn insert(&self, key: K, value: V) { - self.values + fn insert(&self, key: K, value: V) { + self.inner + .values .lock() - .await + .expect("cache map poisoned") .insert(key, (Instant::now(), value)); } - async fn get(&self, key: K) -> Option { - self.values + fn get(&self, key: &K) -> Option { + self.lookup(key).map(|(_, value)| value) + } + + /// Claim the compute slot for `key`: the holder gets the sender (waiters + /// wake when it drops); if already claimed, the receiver to wait on. + fn claim(&self, key: K) -> Result, watch::Receiver<()>> { + let mut inflight = self.inner.inflight.lock().expect("cache map poisoned"); + if let Some(receiver) = inflight.get(&key) { + return Err(receiver.clone()); + } + let (sender, receiver) = watch::channel(()); + inflight.insert(key, receiver); + Ok(sender) + } + + fn release(&self, key: &K) { + self.inner + .inflight .lock() - .await - .get(&key) - .map(|(_, value)| value.clone()) + .expect("cache map poisoned") + .remove(key); } } #[derive(Debug)] -struct RuntimeState { +pub struct RuntimeState { read_only: bool, tail_enabled: bool, tail_interval_secs: u64, @@ -165,23 +372,36 @@ struct RuntimeState { } #[derive(Debug, Default, Clone, Serialize, ToSchema)] -struct RuntimeSnapshot { - tail_running: bool, - tail_running_count: u64, - tail_last_ok_at: Option>, - tail_last_block: Option, - tail_last_rows: Option, - tail_last_error_at: Option>, - tail_last_error: Option, +pub struct RuntimeSnapshot { + pub tail_running: bool, + pub tail_running_count: u64, + pub tail_last_ok_at: Option>, + pub tail_last_block: Option, + pub tail_last_rows: Option, + pub tail_last_error_at: Option>, + pub tail_last_error: Option, + pub tail_chains: Vec, +} + +#[derive(Debug, Default, Clone, Serialize, ToSchema)] +pub struct ChainRuntimeSnapshot { + pub chain_id: u64, + pub tail_running: bool, + pub tail_running_count: u64, + pub tail_last_ok_at: Option>, + pub tail_last_block: Option, + pub tail_last_rows: Option, + pub tail_last_error_at: Option>, + pub tail_last_error: Option, } #[derive(Debug, Serialize, ToSchema)] -struct RuntimeResponse { - read_only: bool, - tail_enabled: bool, - tail_interval_secs: u64, +pub struct RuntimeResponse { + pub read_only: bool, + pub tail_enabled: bool, + pub tail_interval_secs: u64, #[serde(flatten)] - snapshot: RuntimeSnapshot, + pub snapshot: RuntimeSnapshot, } struct TailLoopConfig { @@ -190,11 +410,20 @@ struct TailLoopConfig { confirmations: u64, batch_size: usize, max_concurrent: usize, - data_dir: std::path::PathBuf, + data_dir: PathBuf, +} + +struct TailLoopSettings { + rpcs: Vec, + interval: Duration, + confirmations: u64, + batch_size: usize, + max_concurrent: usize, + data_dir: PathBuf, } impl RuntimeState { - fn new(read_only: bool, tail_enabled: bool, tail_interval_secs: u64) -> Self { + pub fn new(read_only: bool, tail_enabled: bool, tail_interval_secs: u64) -> Self { Self { read_only, tail_enabled, @@ -203,7 +432,7 @@ impl RuntimeState { } } - async fn response(&self) -> RuntimeResponse { + pub async fn response(&self) -> RuntimeResponse { RuntimeResponse { read_only: self.read_only, tail_enabled: self.tail_enabled, @@ -212,31 +441,124 @@ impl RuntimeState { } } - async fn mark_tail_start(&self) { + pub async fn mark_tail_start(&self, chain_id: Option) { let mut snapshot = self.snapshot.lock().await; - snapshot.tail_running_count = snapshot.tail_running_count.saturating_add(1); - snapshot.tail_running = true; + if updates_default_runtime(chain_id) && snapshot.tail_last_ok_at.is_none() { + snapshot.tail_running_count = snapshot.tail_running_count.saturating_add(1); + snapshot.tail_running = true; + } + if let Some(chain_id) = chain_id { + let chain = chain_runtime_snapshot_mut(&mut snapshot, chain_id); + if chain.tail_last_ok_at.is_none() { + chain.tail_running_count = chain.tail_running_count.saturating_add(1); + chain.tail_running = true; + } + } } - async fn mark_tail_ok(&self, end_block: Option, rows: u64) { + pub async fn mark_tail_ok(&self, chain_id: Option, end_block: Option, rows: u64) { let mut snapshot = self.snapshot.lock().await; - snapshot.tail_running_count = snapshot.tail_running_count.saturating_sub(1); - snapshot.tail_running = snapshot.tail_running_count > 0; - snapshot.tail_last_ok_at = Some(Utc::now()); - if let Some(block) = end_block { - snapshot.tail_last_block = Some(block); + let now = Some(Utc::now()); + if updates_default_runtime(chain_id) { + snapshot.tail_running_count = snapshot.tail_running_count.saturating_sub(1); + snapshot.tail_running = snapshot.tail_running_count > 0; + snapshot.tail_last_ok_at = now; + if let Some(block) = end_block { + snapshot.tail_last_block = Some(block); + } + snapshot.tail_last_rows = Some(rows); + snapshot.tail_last_error = None; + snapshot.tail_last_error_at = None; } - snapshot.tail_last_rows = Some(rows); - snapshot.tail_last_error = None; - snapshot.tail_last_error_at = None; + if let Some(chain_id) = chain_id { + let chain = chain_runtime_snapshot_mut(&mut snapshot, chain_id); + chain.tail_running_count = chain.tail_running_count.saturating_sub(1); + chain.tail_running = chain.tail_running_count > 0; + chain.tail_last_ok_at = now; + if let Some(block) = end_block { + chain.tail_last_block = Some(block); + } + chain.tail_last_rows = Some(rows); + chain.tail_last_error = None; + chain.tail_last_error_at = None; + } + } + + pub async fn mark_tail_ready(&self, chain_id: u64, block: Option) { + let mut snapshot = self.snapshot.lock().await; + let now = Some(Utc::now()); + if updates_default_runtime(Some(chain_id)) { + snapshot.tail_last_ok_at = now; + if let Some(block) = block { + snapshot.tail_last_block = Some(block); + } + snapshot.tail_last_rows = Some(0); + snapshot.tail_last_error = None; + snapshot.tail_last_error_at = None; + } + + let chain = chain_runtime_snapshot_mut(&mut snapshot, chain_id); + chain.tail_last_ok_at = now; + chain.tail_last_block = block; + chain.tail_last_rows = Some(0); + chain.tail_last_error = None; + chain.tail_last_error_at = None; } - async fn mark_tail_error(&self, message: String) { + pub async fn mark_tail_error(&self, chain_id: Option, message: String) { let mut snapshot = self.snapshot.lock().await; - snapshot.tail_running_count = snapshot.tail_running_count.saturating_sub(1); - snapshot.tail_running = snapshot.tail_running_count > 0; - snapshot.tail_last_error_at = Some(Utc::now()); - snapshot.tail_last_error = Some(message); + let now = Some(Utc::now()); + if updates_default_runtime(chain_id) { + snapshot.tail_running_count = snapshot.tail_running_count.saturating_sub(1); + snapshot.tail_running = snapshot.tail_running_count > 0; + snapshot.tail_last_error_at = now; + snapshot.tail_last_error = Some(message.clone()); + } + if let Some(chain_id) = chain_id { + let chain = chain_runtime_snapshot_mut(&mut snapshot, chain_id); + chain.tail_running_count = chain.tail_running_count.saturating_sub(1); + chain.tail_running = chain.tail_running_count > 0; + chain.tail_last_error_at = now; + chain.tail_last_error = Some(message); + } + } +} + +fn updates_default_runtime(chain_id: Option) -> bool { + chain_id + .map(|chain_id| chain_id == chains::default_chain_id()) + .unwrap_or(true) +} + +fn chain_runtime_snapshot_mut( + snapshot: &mut RuntimeSnapshot, + chain_id: u64, +) -> &mut ChainRuntimeSnapshot { + if let Some(index) = snapshot + .tail_chains + .iter() + .position(|chain| chain.chain_id == chain_id) + { + return &mut snapshot.tail_chains[index]; + } + snapshot.tail_chains.push(ChainRuntimeSnapshot { + chain_id, + ..ChainRuntimeSnapshot::default() + }); + let index = snapshot.tail_chains.len() - 1; + &mut snapshot.tail_chains[index] +} + +async fn seed_runtime_snapshot(db: &Db, runtime: &RuntimeState) { + for chain in chains::supported_chains() { + match db.highest_contract_block(chain.chain_id).await { + Ok(block) => runtime.mark_tail_ready(chain.chain_id, block).await, + Err(err) => tracing::warn!( + "could not seed runtime state for chain_id={}: {:#}", + chain.chain_id, + err + ), + } } } @@ -246,7 +568,16 @@ pub async fn run_serve(args: ServeArgs) -> Result<()> { "--read-only is set; ignoring --rpc values (background extraction requires a write lock)" ); } - let db = Db::open_with_mode(&args.data_dir, &args.contracts_glob, args.read_only)?; + let db = Db::open( + &args.data_dir, + &args.contracts_glob, + DbOptions { + read_only: args.read_only, + memory_limit: args.db_memory_limit.clone(), + threads: args.db_threads, + readers: args.db_readers, + }, + )?; let rpcs = args .rpc .iter() @@ -261,41 +592,29 @@ pub async fn run_serve(args: ServeArgs) -> Result<()> { args.tail_interval_secs.max(15), )); let cache = Arc::new(ApiCache::default()); + seed_runtime_snapshot(&db, &runtime).await; + // Rollup-backed queries are fast enough to serve cold, so warming happens + // off the startup path — the listener binds immediately. + tokio::spawn(prewarm_initial_dashboard_cache(db.clone(), cache.clone())); + if !args.read_only { + // Materialized SQL-explorer table; queries fall back to a live join + // until it's ready. + let db_explorer = db.clone(); + tokio::spawn(async move { + match db_explorer.refresh_explorer().await { + Ok(true) => tracing::info!("sql explorer table ready"), + Ok(false) => {} + Err(err) => tracing::warn!("sql explorer table rebuild failed: {:#}", err), + } + }); + } let state = AppState { db: db.clone(), runtime: runtime.clone(), cache: cache.clone(), + latency: Arc::new(LatencyStats::default()), }; - prewarm_initial_dashboard_cache(db.clone(), cache.clone()).await; - { - let db_warm = db.clone(); - let cache_warm = cache.clone(); - tokio::spawn(async move { - tokio::time::sleep(Duration::from_secs(5)).await; - prewarm_extended_dashboard_cache(db_warm, cache_warm).await; - }); - } - - if !args.read_only { - for rpc in rpcs { - let db_bg = db.clone(); - let config = TailLoopConfig { - rpc, - interval: Duration::from_secs(args.tail_interval_secs.max(15)), - confirmations: args.tail_confirmations, - batch_size: args.tail_batch_size, - max_concurrent: args.tail_max_concurrent_requests, - data_dir: args.data_dir.clone(), - }; - let runtime_bg = runtime.clone(); - let cache_bg = cache.clone(); - tokio::spawn(async move { - background_tail_loop(db_bg, config, runtime_bg, cache_bg).await; - }); - } - } - let (api_router, api) = OpenApiRouter::with_openapi(ApiDoc::openapi()) .routes(routes!(stats_handler)) .routes(routes!(chains_handler)) @@ -315,8 +634,10 @@ pub async fn run_serve(args: ServeArgs) -> Result<()> { let app = api_router .route("/openapi.json", get(|| async { openapi_json })) .merge(Scalar::with_url("/scalar", api)) + .layer(CompressionLayer::new()) .layer(dashboard_cors_layer()) .layer(TraceLayer::new_for_http()) + .layer(middleware::from_fn_with_state(state.clone(), track_latency)) .with_state(state); let addr: SocketAddr = args @@ -328,6 +649,21 @@ pub async fn run_serve(args: ServeArgs) -> Result<()> { .with_context(|| format!("bind {}", addr))?; tracing::info!("serving blink dashboard on http://{}", addr); tracing::info!(" data dir: {}", args.data_dir.display()); + if !args.read_only { + spawn_tail_loops( + db.clone(), + runtime.clone(), + cache.clone(), + TailLoopSettings { + rpcs, + interval: Duration::from_secs(args.tail_interval_secs.max(15)), + confirmations: args.tail_confirmations, + batch_size: args.tail_batch_size, + max_concurrent: args.tail_max_concurrent_requests, + data_dir: args.data_dir.clone(), + }, + ); + } axum::serve(listener, app) .await .context("axum server failed") @@ -388,6 +724,17 @@ fn selected_chain_id(chain_id: Option) -> u64 { chain_id.unwrap_or_else(chains::default_chain_id) } +async fn cached_highest_block(state: &AppState, chain_id: u64) -> Result { + let db = state.db.clone(); + state + .cache + .highest_blocks + .get_or_refresh(chain_id, HIGHEST_BLOCK_TTL, move || async move { + Ok(db.highest_contract_block(chain_id).await?.unwrap_or(0)) + }) + .await +} + #[derive(Serialize, ToSchema)] struct ChainsResponse { chains: Vec, @@ -422,53 +769,136 @@ async fn stats_handler( Query(q): Query, ) -> Result, AppError> { let chain_id = selected_chain_id(q.chain_id); + let db = state.db.clone(); let stats = state .cache .stats - .get_or_try_update(chain_id, API_CACHE_TTL, state.db.stats(chain_id)) + .get_or_refresh(chain_id, API_CACHE_TTL, move || async move { + db.stats(chain_id).await + }) .await?; Ok(Json(stats)) } +#[derive(Serialize, ToSchema)] +struct RuntimeApiResponse { + #[serde(flatten)] + runtime: RuntimeResponse, + /// Rolling latency of every `/api/*` request since startup. + api_latency: LatencySnapshot, +} + #[utoipa::path( get, path = "/api/runtime", tag = API_TAG, - responses((status = OK, body = RuntimeResponse)) + responses((status = OK, body = RuntimeApiResponse)) )] -async fn runtime_handler(State(state): State) -> Json { - Json(state.runtime.response().await) +async fn runtime_handler(State(state): State) -> Json { + Json(RuntimeApiResponse { + runtime: state.runtime.response().await, + api_latency: state.latency.snapshot(), + }) } #[derive(Default, Deserialize, IntoParams)] #[into_params(parameter_in = Query)] -struct BucketQuery { +pub struct BucketQuery { /// Chain ID to query. Defaults to Ethereum mainnet (1). - chain_id: Option, + pub chain_id: Option, /// Optional internal aggregation bucket: `hour`, `day`, `week`, `month`, `year`, or raw block count. #[serde(default)] - bucket: Option, + pub bucket: Option, /// Visible time range: `hour`, `day`, `week`, `month`, or `year`. #[serde(default)] - range: Option, + pub range: Option, /// Optional block number where the visible range should end. Defaults to the latest indexed block. #[serde(default)] - end_block: Option, + pub end_block: Option, /// Optional block number where the visible range should start. #[serde(default)] - start_block: Option, + pub start_block: Option, /// Optional ISO-8601 timestamp where the visible range should start. #[serde(default)] - start_time: Option>, + pub start_time: Option>, /// Optional ISO-8601 timestamp where the visible range should end. #[serde(default)] - end_time: Option>, + pub end_time: Option>, + /// Maximum number of compiler versions to return. + #[serde(default)] + pub limit: Option, } -#[derive(Clone, Copy)] -struct TimeSeriesWindow { +fn relative_range_code(range: &str) -> u64 { + match range { + "hour" => 1, + "day" => 2, + "week" => 3, + "month" => 4, + "year" => 5, + _ => 0, + } +} + +fn uses_relative_preset_window(q: &BucketQuery) -> bool { + q.range.is_some() + && q.start_block.is_none() + && q.end_block.is_none() + && q.start_time.is_none() + && q.end_time.is_none() +} + +fn normalized_cache_range( + q: &BucketQuery, + window: TimeSeriesWindow, bucket_blocks: u64, - block_range: Option<(u64, u64)>, +) -> (Option, Option, Option) { + if uses_relative_preset_window(q) { + let range_code = q.range.as_deref().map(relative_range_code); + let end_bucket = window + .block_range + .map(|(_, end)| end / bucket_blocks.max(1)); + return (Some(bucket_blocks.max(1)), range_code, end_bucket); + } + ( + None, + window.block_range.map(|(start, _)| start), + window.block_range.map(|(_, end)| end), + ) +} + +pub fn bucket_cache_key( + chain_id: u64, + q: &BucketQuery, + window: TimeSeriesWindow, +) -> BucketCacheKey { + let (_, start_block, end_block) = normalized_cache_range(q, window, window.bucket_blocks); + BucketCacheKey { + chain_id, + bucket: window.bucket_blocks, + start_block, + end_block, + } +} + +pub fn range_cache_key_for_query( + chain_id: u64, + q: &BucketQuery, + window: TimeSeriesWindow, +) -> RangeCacheKey { + let (bucket, start_block, end_block) = normalized_cache_range(q, window, window.bucket_blocks); + RangeCacheKey { + chain_id, + bucket, + start_block, + end_block, + } +} + +#[derive(Clone, Copy)] +pub struct TimeSeriesWindow { + pub bucket_blocks: u64, + pub block_range: Option<(u64, u64)>, } fn parse_bucket_value(bucket: Option<&str>, chain_id: u64, anchor_block: u64) -> u64 { @@ -484,7 +914,11 @@ fn parse_bucket_value(bucket: Option<&str>, chain_id: u64, anchor_block: u64) -> } } -fn parse_time_series_window(q: &BucketQuery, chain_id: u64, anchor_block: u64) -> TimeSeriesWindow { +pub fn parse_time_series_window( + q: &BucketQuery, + chain_id: u64, + anchor_block: u64, +) -> TimeSeriesWindow { let blocks_per_day = blocks_per_day(chain_id, anchor_block).max(1); let explicit_end = q.end_block.or_else(|| { q.end_time @@ -543,6 +977,17 @@ fn parse_time_series_window(q: &BucketQuery, chain_id: u64, anchor_block: u64) - } } +fn default_aggregate_window(q: &mut BucketQuery) { + if q.range.is_none() + && q.start_block.is_none() + && q.end_block.is_none() + && q.start_time.is_none() + && q.end_time.is_none() + { + q.range = Some(INITIAL_AGGREGATE_RANGE.to_string()); + } +} + #[utoipa::path( get, path = "/api/deploys-over-time", @@ -558,28 +1003,17 @@ async fn deploys_handler( Query(q): Query, ) -> Result, AppError> { let chain_id = selected_chain_id(q.chain_id); - let highest = state - .db - .highest_contract_block(chain_id) - .await? - .unwrap_or(0); + let highest = cached_highest_block(&state, chain_id).await?; let window = parse_time_series_window(&q, chain_id, highest); - let cache_key = BucketCacheKey { - chain_id, - bucket: window.bucket_blocks, - start_block: window.block_range.map(|(start, _)| start), - end_block: window.block_range.map(|(_, end)| end), - }; + let cache_key = bucket_cache_key(chain_id, &q, window); + let db = state.db.clone(); let buckets = state .cache .deploys - .get_or_try_update( - cache_key, - API_CACHE_TTL, - state - .db - .deploys_over_time(chain_id, window.bucket_blocks, window.block_range), - ) + .get_or_refresh(cache_key, API_CACHE_TTL, move || async move { + db.deploys_over_time(chain_id, window.bucket_blocks, window.block_range) + .await + }) .await?; Ok(Json(DeploysResponse { bucket_blocks: window.bucket_blocks, @@ -605,28 +1039,17 @@ async fn verified_handler( Query(q): Query, ) -> Result, AppError> { let chain_id = selected_chain_id(q.chain_id); - let highest = state - .db - .highest_contract_block(chain_id) - .await? - .unwrap_or(0); + let highest = cached_highest_block(&state, chain_id).await?; let window = parse_time_series_window(&q, chain_id, highest); - let cache_key = BucketCacheKey { - chain_id, - bucket: window.bucket_blocks, - start_block: window.block_range.map(|(start, _)| start), - end_block: window.block_range.map(|(_, end)| end), - }; + let cache_key = bucket_cache_key(chain_id, &q, window); + let db = state.db.clone(); let buckets = state .cache .verified - .get_or_try_update( - cache_key, - API_CACHE_TTL, - state - .db - .verified_ratio_over_time(chain_id, window.bucket_blocks, window.block_range), - ) + .get_or_refresh(cache_key, API_CACHE_TTL, move || async move { + db.verified_ratio_over_time(chain_id, window.bucket_blocks, window.block_range) + .await + }) .await?; Ok(Json(VerifiedResponse { bucket_blocks: window.bucket_blocks, @@ -641,7 +1064,7 @@ async fn verified_handler( get, path = "/api/bytecode-sizes", tag = API_TAG, - params(ChainQuery), + params(BucketQuery), responses( (status = OK, body = SizeResponse), (status = INTERNAL_SERVER_ERROR, body = ApiError) @@ -649,30 +1072,25 @@ async fn verified_handler( )] async fn bytecode_sizes_handler( State(state): State, - Query(q): Query, + Query(mut q): Query, ) -> Result, AppError> { let chain_id = selected_chain_id(q.chain_id); + default_aggregate_window(&mut q); + let highest = cached_highest_block(&state, chain_id).await?; + let window = parse_time_series_window(&q, chain_id, highest); + let cache_key = range_cache_key_for_query(chain_id, &q, window); + let db = state.db.clone(); let bins_out = state .cache .bytecode_sizes - .get_or_try_update( - chain_id, - API_CACHE_TTL, - state.db.bytecode_size_distribution(chain_id), - ) + .get_or_refresh(cache_key, API_CACHE_TTL, move || async move { + db.bytecode_size_distribution(chain_id, window.block_range) + .await + }) .await?; Ok(Json(SizeResponse { bins: bins_out })) } -#[derive(Deserialize, IntoParams)] -#[into_params(parameter_in = Query)] -struct LimitQuery { - /// Chain ID to query. Defaults to Ethereum mainnet (1). - chain_id: Option, - /// Maximum number of compiler versions to return. - limit: Option, -} - #[derive(Deserialize, IntoParams)] #[into_params(parameter_in = Query)] struct PageQuery { @@ -690,7 +1108,7 @@ struct PageQuery { get, path = "/api/compilers", tag = API_TAG, - params(LimitQuery), + params(BucketQuery), responses( (status = OK, body = CompilersResponse), (status = INTERNAL_SERVER_ERROR, body = ApiError) @@ -698,18 +1116,31 @@ struct PageQuery { )] async fn compilers_handler( State(state): State, - Query(q): Query, + Query(mut q): Query, ) -> Result, AppError> { let chain_id = selected_chain_id(q.chain_id); - let limit = q.limit.unwrap_or(15); - let cache_key = LimitCacheKey { chain_id, limit }; + default_aggregate_window(&mut q); + let limit = q.limit.unwrap_or(DEFAULT_COMPILER_LIMIT); + let highest = cached_highest_block(&state, chain_id).await?; + let window = parse_time_series_window(&q, chain_id, highest); + let (bucket, start_block, end_block) = normalized_cache_range(&q, window, window.bucket_blocks); + let cache_key = LimitRangeCacheKey { + chain_id, + limit, + bucket, + start_block, + end_block, + }; + let db = state.db.clone(); let (compilers, total_known) = state .cache .compilers - .get_or_try_update(cache_key, API_CACHE_TTL, async { + .get_or_refresh(cache_key, API_CACHE_TTL, move || async move { Ok(( - state.db.top_compilers(chain_id, limit).await?, - state.db.compiler_version_total(chain_id).await?, + db.top_compilers(chain_id, limit, window.block_range) + .await?, + db.compiler_version_total(chain_id, window.block_range) + .await?, )) }) .await?; @@ -748,14 +1179,13 @@ async fn recent_handler( }), _ => None, }; + let db = state.db.clone(); let page = match state .cache .recent - .get_or_try_update( - cache_key, - API_CACHE_TTL, - state.db.recent_contracts(chain_id, limit, cursor), - ) + .get_or_refresh(cache_key, API_CACHE_TTL, move || async move { + db.recent_contracts(chain_id, limit, cursor).await + }) .await { Ok(page) => page, @@ -764,8 +1194,7 @@ async fn recent_handler( state .cache .recent - .get(cache_key) - .await + .get(&cache_key) .unwrap_or(crate::db::RecentPage { contracts: Vec::new(), has_more: false, @@ -831,14 +1260,13 @@ async fn languages_handler( Query(q): Query, ) -> Result, AppError> { let chain_id = selected_chain_id(q.chain_id); + let db = state.db.clone(); let languages = state .cache .languages - .get_or_try_update( - chain_id, - API_CACHE_TTL, - state.db.language_distribution(chain_id), - ) + .get_or_refresh(chain_id, API_CACHE_TTL, move || async move { + db.language_distribution(chain_id).await + }) .await?; Ok(Json(LanguagesResponse { languages })) } @@ -847,7 +1275,7 @@ async fn languages_handler( get, path = "/api/standards", tag = API_TAG, - params(ChainQuery), + params(BucketQuery), responses( (status = OK, body = crate::db::StandardsBreakdown), (status = INTERNAL_SERVER_ERROR, body = ApiError) @@ -855,17 +1283,20 @@ async fn languages_handler( )] async fn standards_handler( State(state): State, - Query(q): Query, + Query(mut q): Query, ) -> Result, AppError> { let chain_id = selected_chain_id(q.chain_id); + default_aggregate_window(&mut q); + let highest = cached_highest_block(&state, chain_id).await?; + let window = parse_time_series_window(&q, chain_id, highest); + let cache_key = range_cache_key_for_query(chain_id, &q, window); + let db = state.db.clone(); let standards = state .cache .standards - .get_or_try_update( - chain_id, - API_CACHE_TTL, - state.db.standards_breakdown(chain_id), - ) + .get_or_refresh(cache_key, API_CACHE_TTL, move || async move { + db.standards_breakdown(chain_id, window.block_range).await + }) .await?; Ok(Json(standards)) } @@ -913,6 +1344,7 @@ struct LanguagesResponse { async fn prewarm_initial_dashboard_cache(db: Db, cache: Arc) { let started = Instant::now(); + tracing::info!("warming dashboard cache in background"); for chain in chains::supported_chains() { prewarm_chain_dashboard_cache( &db, @@ -924,21 +1356,35 @@ async fn prewarm_initial_dashboard_cache(db: Db, cache: Arc) { .await; } tracing::info!( - "initial dashboard cache warmed in {:.1}s", + "dashboard cache warmed in {:.1}s", started.elapsed().as_secs_f64() ); } -async fn prewarm_extended_dashboard_cache(db: Db, cache: Arc) { - let started = Instant::now(); - for chain in chains::supported_chains() { - prewarm_chain_dashboard_cache(&db, &cache, chain.chain_id, PRESET_CHART_RANGES, false) - .await; +fn spawn_tail_loops( + db: Db, + runtime: Arc, + cache: Arc, + settings: TailLoopSettings, +) { + for rpc in settings.rpcs { + let db_bg = db.clone(); + let runtime_bg = runtime.clone(); + let cache_bg = cache.clone(); + let data_dir = settings.data_dir.clone(); + let config = TailLoopConfig { + rpc, + interval: settings.interval, + confirmations: settings.confirmations, + batch_size: settings.batch_size, + max_concurrent: settings.max_concurrent, + data_dir, + }; + tokio::spawn(async move { + tokio::time::sleep(TAIL_START_DELAY).await; + background_tail_loop(db_bg, config, runtime_bg, cache_bg).await; + }); } - tracing::info!( - "extended chart cache warmed in {:.1}s", - started.elapsed().as_secs_f64() - ); } async fn prewarm_chain_dashboard_cache( @@ -957,41 +1403,74 @@ async fn prewarm_chain_dashboard_cache( return; } }; + cache.highest_blocks.insert(chain_id, highest); + + for range in chart_ranges { + prewarm_chart_range(db, cache, chain_id, highest, range).await; + } if include_widgets { + let aggregate_query = BucketQuery { + chain_id: Some(chain_id), + range: Some(INITIAL_AGGREGATE_RANGE.to_string()), + ..BucketQuery::default() + }; + let aggregate_window = parse_time_series_window(&aggregate_query, chain_id, highest); + let aggregate_key = range_cache_key_for_query(chain_id, &aggregate_query, aggregate_window); + match db.stats(chain_id).await { - Ok(stats) => cache.stats.insert(chain_id, stats).await, + Ok(stats) => cache.stats.insert(chain_id, stats), Err(err) => log_prewarm_error(chain_id, "stats", err), } - match db.bytecode_size_distribution(chain_id).await { - Ok(bins) => cache.bytecode_sizes.insert(chain_id, bins).await, + match db + .bytecode_size_distribution(chain_id, aggregate_window.block_range) + .await + { + Ok(bins) => cache.bytecode_sizes.insert(aggregate_key, bins), Err(err) => log_prewarm_error(chain_id, "bytecode sizes", err), } - let compiler_key = LimitCacheKey { + let (compiler_bucket, compiler_start_block, compiler_end_block) = normalized_cache_range( + &aggregate_query, + aggregate_window, + aggregate_window.bucket_blocks, + ); + let compiler_key = LimitRangeCacheKey { chain_id, limit: DEFAULT_COMPILER_LIMIT, + bucket: compiler_bucket, + start_block: compiler_start_block, + end_block: compiler_end_block, }; match async { Ok::<_, anyhow::Error>(( - db.top_compilers(chain_id, DEFAULT_COMPILER_LIMIT).await?, - db.compiler_version_total(chain_id).await?, + db.top_compilers( + chain_id, + DEFAULT_COMPILER_LIMIT, + aggregate_window.block_range, + ) + .await?, + db.compiler_version_total(chain_id, aggregate_window.block_range) + .await?, )) } .await { - Ok(compilers) => cache.compilers.insert(compiler_key, compilers).await, + Ok(compilers) => cache.compilers.insert(compiler_key, compilers), Err(err) => log_prewarm_error(chain_id, "compilers", err), } match db.language_distribution(chain_id).await { - Ok(languages) => cache.languages.insert(chain_id, languages).await, + Ok(languages) => cache.languages.insert(chain_id, languages), Err(err) => log_prewarm_error(chain_id, "languages", err), } - match db.standards_breakdown(chain_id).await { - Ok(standards) => cache.standards.insert(chain_id, standards).await, + match db + .standards_breakdown(chain_id, aggregate_window.block_range) + .await + { + Ok(standards) => cache.standards.insert(aggregate_key, standards), Err(err) => log_prewarm_error(chain_id, "standards", err), } @@ -1005,15 +1484,11 @@ async fn prewarm_chain_dashboard_cache( .recent_contracts(chain_id, DEFAULT_RECENT_LIMIT, None) .await { - Ok(recent) => cache.recent.insert(recent_key, recent).await, + Ok(recent) => cache.recent.insert(recent_key, recent), Err(err) => log_prewarm_error(chain_id, "recent deployments", err), } } - for range in chart_ranges { - prewarm_chart_range(db, cache, chain_id, highest, range).await; - } - tracing::debug!( "dashboard cache warmed for chain_id={} in {:.1}s", chain_id, @@ -1028,18 +1503,13 @@ async fn prewarm_chart_range(db: &Db, cache: &ApiCache, chain_id: u64, highest: ..BucketQuery::default() }; let window = parse_time_series_window(&query, chain_id, highest); - let cache_key = BucketCacheKey { - chain_id, - bucket: window.bucket_blocks, - start_block: window.block_range.map(|(start, _)| start), - end_block: window.block_range.map(|(_, end)| end), - }; + let cache_key = bucket_cache_key(chain_id, &query, window); match db .deploys_over_time(chain_id, window.bucket_blocks, window.block_range) .await { - Ok(buckets) => cache.deploys.insert(cache_key, buckets).await, + Ok(buckets) => cache.deploys.insert(cache_key, buckets), Err(err) => log_prewarm_error(chain_id, &format!("deployments {range}"), err), } @@ -1047,7 +1517,7 @@ async fn prewarm_chart_range(db: &Db, cache: &ApiCache, chain_id: u64, highest: .verified_ratio_over_time(chain_id, window.bucket_blocks, window.block_range) .await { - Ok(buckets) => cache.verified.insert(cache_key, buckets).await, + Ok(buckets) => cache.verified.insert(cache_key, buckets), Err(err) => log_prewarm_error(chain_id, &format!("verification {range}"), err), } } @@ -1088,7 +1558,7 @@ async fn background_tail_loop( config.confirmations ); loop { - runtime.mark_tail_start().await; + runtime.mark_tail_start(chain_id).await; match crate::extract::tail::tail_once( &db, &config.rpc, @@ -1101,134 +1571,42 @@ async fn background_tail_loop( { Ok(Some(report)) => { runtime - .mark_tail_ok(Some(report.end_block), report.rows as u64) + .mark_tail_ok(chain_id, Some(report.end_block), report.rows as u64) .await; + tracing::info!( + "tail extracted blocks {}-{} ({} contracts)", + report.start_block, + report.end_block, + report.rows + ); + // New blocks just landed in the rollups: overwrite this + // chain's cached dashboard entries right away instead of + // letting the top-right block number and the recent table + // trail by up to a cache TTL. These are rollup queries — + // milliseconds — so doing it every tick is cheap. if let Some(chain_id) = chain_id { - let db_refresh = db.clone(); - let cache_refresh = cache.clone(); - tokio::spawn(async move { + if report.rows > 0 { prewarm_chain_dashboard_cache( - &db_refresh, - &cache_refresh, + &db, + &cache, chain_id, - PRESET_CHART_RANGES, + &[INITIAL_DEPLOYS_RANGE, INITIAL_VERIFIED_RANGE], true, ) .await; - }); + } } - tracing::info!( - "tail extracted blocks {}-{} ({} contracts)", - report.start_block, - report.end_block, - report.rows - ); } Ok(None) => { - runtime.mark_tail_ok(None, 0).await; + runtime.mark_tail_ok(chain_id, None, 0).await; tracing::debug!("tail: no new blocks"); } Err(err) => { let msg = format!("{:#}", err); - runtime.mark_tail_error(msg.clone()).await; + runtime.mark_tail_error(chain_id, msg.clone()).await; tracing::warn!("tail failed: {}", msg); } } tokio::time::sleep(config.interval).await; } } - -#[cfg(test)] -mod tests { - use crate::chains::{ETHEREUM_CHAIN_ID, GNOSIS_CHAIN_ID}; - - use super::{parse_time_series_window, BucketQuery}; - - fn query(range: Option<&str>, bucket: Option<&str>) -> BucketQuery { - BucketQuery { - chain_id: None, - bucket: bucket.map(str::to_string), - range: range.map(str::to_string), - end_block: None, - start_block: None, - start_time: None, - end_time: None, - } - } - - #[test] - fn day_range_limits_chart_to_last_day_with_hourly_buckets() { - let anchor_block = 20_000_000; - let window = - parse_time_series_window(&query(Some("day"), None), ETHEREUM_CHAIN_ID, anchor_block); - - assert_eq!(window.block_range, Some((19_992_801, 20_000_000))); - assert_eq!(window.bucket_blocks, 300); - } - - #[test] - fn week_range_limits_chart_to_last_week_with_daily_buckets() { - let anchor_block = 20_000_000; - let window = - parse_time_series_window(&query(Some("week"), None), ETHEREUM_CHAIN_ID, anchor_block); - - assert_eq!(window.block_range, Some((19_949_601, 20_000_000))); - assert_eq!(window.bucket_blocks, 7_200); - } - - #[test] - fn year_range_limits_chart_to_last_year_with_monthly_buckets() { - let anchor_block = 20_000_000; - let window = - parse_time_series_window(&query(Some("year"), None), ETHEREUM_CHAIN_ID, anchor_block); - - assert_eq!(window.block_range, Some((17_372_001, 20_000_000))); - assert_eq!(window.bucket_blocks, 216_000); - } - - #[test] - fn hour_range_uses_chain_specific_block_time() { - let anchor_block = 46_000_000; - let window = - parse_time_series_window(&query(Some("hour"), None), GNOSIS_CHAIN_ID, anchor_block); - - assert_eq!(window.block_range, Some((45_999_281, 46_000_000))); - assert_eq!(window.bucket_blocks, 60); - } - - #[test] - fn legacy_bucket_query_keeps_full_history_behavior() { - let anchor_block = 20_000_000; - let window = - parse_time_series_window(&query(None, Some("day")), ETHEREUM_CHAIN_ID, anchor_block); - - assert_eq!(window.block_range, None); - assert_eq!(window.bucket_blocks, 7_200); - } - - #[test] - fn range_end_block_moves_visible_window() { - let anchor_block = 20_000_000; - let mut q = query(Some("day"), None); - q.end_block = Some(19_000_000); - let window = parse_time_series_window(&q, ETHEREUM_CHAIN_ID, anchor_block); - - assert_eq!(window.block_range, Some((18_992_801, 19_000_000))); - - q.end_block = Some(21_000_000); - let capped = parse_time_series_window(&q, ETHEREUM_CHAIN_ID, anchor_block); - assert_eq!(capped.block_range, Some((19_992_801, 20_000_000))); - } - - #[test] - fn explicit_start_block_creates_custom_window() { - let anchor_block = 20_000_000; - let mut q = query(None, None); - q.start_block = Some(19_900_000); - q.end_block = Some(19_950_000); - let window = parse_time_series_window(&q, ETHEREUM_CHAIN_ID, anchor_block); - - assert_eq!(window.block_range, Some((19_900_000, 19_950_000))); - assert_eq!(window.bucket_blocks, 520); - } -} diff --git a/tests/blocks.rs b/tests/blocks.rs new file mode 100644 index 0000000..1ebdf7c --- /dev/null +++ b/tests/blocks.rs @@ -0,0 +1,29 @@ +//! Block <-> timestamp mapping tests. + +use blink::blocks::{block_number_at_time, block_timestamp}; +use blink::chains::GNOSIS_CHAIN_ID; +use chrono::{Datelike, TimeZone}; + +#[test] +fn gnosis_recent_blocks_map_to_current_calendar_dates() { + assert_eq!( + block_timestamp(GNOSIS_CHAIN_ID, 46_762_380), + chrono::Utc + .with_ymd_and_hms(2026, 6, 18, 16, 0, 38) + .unwrap() + ); +} + +#[test] +fn gnosis_recent_time_ranges_convert_back_to_blocks() { + let block = block_number_at_time( + GNOSIS_CHAIN_ID, + chrono::Utc.with_ymd_and_hms(2026, 6, 19, 0, 0, 0).unwrap(), + ); + + assert!(block > 46_762_380); + assert_eq!( + block_timestamp(GNOSIS_CHAIN_ID, block).date_naive().month(), + 6 + ); +} diff --git a/tests/db_dashboard.rs b/tests/db_dashboard.rs new file mode 100644 index 0000000..a03a2a9 --- /dev/null +++ b/tests/db_dashboard.rs @@ -0,0 +1,773 @@ +//! Dashboard query-layer tests: rollup ingest correctness (dedup across +//! overlapping parquet files, zellic + parquet union), chain filtering, and +//! the read-only SQL endpoint. + +use std::{ + fs, + path::{Path, PathBuf}, + time::{SystemTime, UNIX_EPOCH}, +}; + +use blink::chains::ETHEREUM_CHAIN_ID; +use blink::db::Db; +use duckdb::Connection; + +struct TestDir { + path: PathBuf, +} + +impl TestDir { + fn new(name: &str) -> Self { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let path = std::env::temp_dir().join(format!( + "blink_db_test_{}_{}_{}", + std::process::id(), + name, + unique + )); + fs::create_dir_all(&path).unwrap(); + Self { path } + } +} + +impl Drop for TestDir { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.path); + } +} + +fn make_bytes(value: u8, len: usize) -> Vec { + vec![value; len] +} + +fn insert_zellic_snapshot(data_dir: &Path) { + let conn = Connection::open(data_dir.join("blink.duckdb")).unwrap(); + conn.execute_batch( + r#" + CREATE TABLE zellic_bytecodes ( + code_hash BLOB, + code BLOB, + n_code_bytes UINTEGER + ); + CREATE TABLE zellic_contracts ( + contract_address BLOB, + bytecode_hash BLOB, + block_number UINTEGER, + create_index UINTEGER, + chain_id UBIGINT + ); + "#, + ) + .unwrap(); + conn.execute( + "INSERT INTO zellic_bytecodes VALUES (?, ?, ?)", + duckdb::params![make_bytes(1, 32), make_bytes(0x60, 4), 4u32], + ) + .unwrap(); + conn.execute( + "INSERT INTO zellic_contracts VALUES (?, ?, ?, ?, ?)", + duckdb::params![make_bytes(2, 20), make_bytes(1, 32), 100u32, 0u32, 1u64], + ) + .unwrap(); +} + +fn write_contract_parquet(path: &Path, block_number: u32, fill: u8, chain_id: u64) { + let path_sql = path.display().to_string().replace('\'', "''"); + let conn = Connection::open_in_memory().unwrap(); + conn.execute_batch(&format!( + r#" + COPY ( + SELECT + {block_number}::UINTEGER AS block_number, + unhex(repeat('{fill:02x}', 32)) AS block_hash, + 0::UINTEGER AS create_index, + unhex(repeat('{fill:02x}', 32)) AS transaction_hash, + unhex(repeat('{addr:02x}', 20)) AS contract_address, + unhex(repeat('{deployer:02x}', 20)) AS deployer, + unhex(repeat('{deployer:02x}', 20)) AS factory, + unhex('6000') AS init_code, + unhex('6001') AS code, + unhex(repeat('{fill:02x}', 32)) AS init_code_hash, + 2::UINTEGER AS n_init_code_bytes, + 2::UINTEGER AS n_code_bytes, + unhex(repeat('{hash:02x}', 32)) AS code_hash, + {chain_id}::UBIGINT AS chain_id + ) TO '{path_sql}' (FORMAT PARQUET); + "#, + fill = fill, + addr = fill.wrapping_add(2), + deployer = fill.wrapping_add(3), + hash = fill.wrapping_add(6), + )) + .unwrap(); +} + +/// Many deployments in one file: one row per block in `blocks`, all sharing +/// one code_hash (`repeat('aa', 32)`) with n_code_bytes = 100. +fn write_multi_block_parquet(path: &Path, blocks: &[u32], chain_id: u64) { + let path_sql = path.display().to_string().replace('\'', "''"); + let block_list = blocks + .iter() + .map(u32::to_string) + .collect::>() + .join(", "); + let conn = Connection::open_in_memory().unwrap(); + conn.execute_batch(&format!( + r#" + COPY ( + SELECT + block_number::UINTEGER AS block_number, + md5(('bh' || block_number)::VARCHAR)::BLOB AS block_hash, + 0::UINTEGER AS create_index, + md5(('tx' || block_number)::VARCHAR)::BLOB AS transaction_hash, + substr(md5(('ad' || block_number)::VARCHAR), 1, 20)::BLOB AS contract_address, + unhex(repeat('11', 20)) AS deployer, + NULL::BLOB AS factory, + unhex('6000') AS init_code, + unhex(repeat('60', 100)) AS code, + md5('ih')::BLOB AS init_code_hash, + 2::UINTEGER AS n_init_code_bytes, + 100::UINTEGER AS n_code_bytes, + unhex(repeat('aa', 32)) AS code_hash, + {chain_id}::UBIGINT AS chain_id + FROM (SELECT unnest([{block_list}]) AS block_number) + ) TO '{path_sql}' (FORMAT PARQUET); + "# + )) + .unwrap(); +} + +fn write_backfill_parquet(data_dir: &Path) { + write_contract_parquet( + &data_dir.join("contracts__0000000200__0000000200.parquet"), + 200, + 0x03, + 1, + ); + write_contract_parquet( + &data_dir.join("contracts__chain_0000000100__0000000300__0000000300.parquet"), + 300, + 0x13, + 100, + ); +} + +fn write_overlapping_ethereum_parquet(data_dir: &Path) { + // The same deployment written into two files: a backfill chunk and a tail + // file. Rollup ingest must count it once. + write_contract_parquet( + &data_dir.join("contracts__0000000250__0000000250.parquet"), + 250, + 0x23, + 1, + ); + write_contract_parquet( + &data_dir.join("tail__chain_0000000001__0000000250__0000000250.parquet"), + 250, + 0x23, + 1, + ); +} + +#[tokio::test] +async fn stats_and_recent_include_parquet_rows_newer_than_zellic() { + let dir = TestDir::new("parquet_newer_than_zellic"); + insert_zellic_snapshot(&dir.path); + write_backfill_parquet(&dir.path); + + let db = Db::open_with_mode(&dir.path, "*.parquet", false).unwrap(); + + let stats = db.stats(ETHEREUM_CHAIN_ID).await.unwrap(); + assert_eq!(stats.total_contracts, 2); + assert_eq!(stats.first_block, 100); + assert_eq!(stats.last_block, 200); + + let deploys = db + .deploys_over_time(ETHEREUM_CHAIN_ID, 100, Some((0, 250))) + .await + .unwrap(); + assert_eq!( + deploys + .iter() + .map(|bucket| (bucket.block_start, bucket.count)) + .collect::>(), + vec![(100, 1), (200, 1)] + ); + + let recent = db + .recent_contracts(ETHEREUM_CHAIN_ID, 5, None) + .await + .unwrap(); + assert_eq!(recent.contracts.len(), 2); + assert_eq!(recent.contracts[0].block_number, 200); + assert_eq!( + recent.contracts[0].address, + format!("0x{}", hex_string(0x05, 20)) + ); + assert_eq!(recent.contracts[1].block_number, 100); + assert!(!recent.has_more); +} + +#[tokio::test] +async fn rollups_are_idempotent_across_reopens() { + let dir = TestDir::new("rollup_idempotent_reopen"); + insert_zellic_snapshot(&dir.path); + write_backfill_parquet(&dir.path); + + { + let db = Db::open_with_mode(&dir.path, "*.parquet", false).unwrap(); + assert_eq!( + db.stats(ETHEREUM_CHAIN_ID).await.unwrap().total_contracts, + 2 + ); + } + // Second open must not re-ingest tracked sources. + let db = Db::open_with_mode(&dir.path, "*.parquet", false).unwrap(); + assert_eq!( + db.stats(ETHEREUM_CHAIN_ID).await.unwrap().total_contracts, + 2 + ); + assert_eq!(db.stats(100).await.unwrap().total_contracts, 1); +} + +#[tokio::test] +async fn refresh_ingests_new_tail_files_incrementally() { + let dir = TestDir::new("refresh_ingests_tail"); + write_backfill_parquet(&dir.path); + + let db = Db::open_with_mode(&dir.path, "*.parquet", false).unwrap(); + assert_eq!( + db.stats(ETHEREUM_CHAIN_ID).await.unwrap().total_contracts, + 1 + ); + + write_contract_parquet( + &dir.path + .join("tail__chain_0000000001__0000000400__0000000400.parquet"), + 400, + 0x33, + 1, + ); + db.refresh().await.unwrap(); + + let stats = db.stats(ETHEREUM_CHAIN_ID).await.unwrap(); + assert_eq!(stats.total_contracts, 2); + assert_eq!(stats.last_block, 400); + assert_eq!( + db.highest_contract_block(ETHEREUM_CHAIN_ID).await.unwrap(), + Some(400) + ); +} + +#[tokio::test] +async fn chart_queries_deduplicate_overlapping_parquet_deployments() { + let dir = TestDir::new("charts_deduplicate_overlapping_parquet"); + write_overlapping_ethereum_parquet(&dir.path); + + let db = Db::open_with_mode(&dir.path, "*.parquet", false).unwrap(); + + let deploys = db + .deploys_over_time(ETHEREUM_CHAIN_ID, 100, Some((250, 250))) + .await + .unwrap(); + assert_eq!(deploys.len(), 1); + assert_eq!(deploys[0].count, 1); + + let verified = db + .verified_ratio_over_time(ETHEREUM_CHAIN_ID, 100, Some((250, 250))) + .await + .unwrap(); + assert_eq!(verified.len(), 1); + assert_eq!(verified[0].verified, 0); + assert_eq!(verified[0].unverified, 0); + assert_eq!(verified[0].unknown, 1); + + db.execute_batch( + r#" + INSERT INTO bytecode_metadata_by_hash ( + code_hash, + language, + compiler_version, + has_source_hash, + is_erc20, + is_erc721, + is_erc1155, + is_proxy_eip1967, + is_proxy_minimal, + uses_push0 + ) VALUES (unhex(repeat('29', 32)), 'solidity', '0.8.20', true, true, false, false, false, false, true) + "# + .to_string(), + ) + .await + .unwrap(); + + let sizes = db + .bytecode_size_distribution(ETHEREUM_CHAIN_ID, Some((250, 250))) + .await + .unwrap(); + assert_eq!(sizes.iter().map(|bin| bin.count).sum::(), 1); + assert_eq!( + sizes + .iter() + .find(|bin| bin.label == "1-32 B") + .map(|bin| bin.count), + Some(1) + ); + + let compilers = db + .top_compilers(ETHEREUM_CHAIN_ID, 12, Some((250, 250))) + .await + .unwrap(); + assert_eq!(compilers.len(), 1); + assert_eq!(compilers[0].compiler_version, "0.8.20"); + assert_eq!(compilers[0].count, 1); + assert_eq!( + db.compiler_version_total(ETHEREUM_CHAIN_ID, Some((250, 250))) + .await + .unwrap(), + 1 + ); + + let standards = db + .standards_breakdown(ETHEREUM_CHAIN_ID, Some((250, 250))) + .await + .unwrap(); + + assert_eq!(standards.total_decoded, 1); + assert_eq!(standards.erc20, 1); + assert_eq!(standards.uses_push0, 1); + assert_eq!(standards.has_source_hash, 1); +} + +#[tokio::test] +async fn chart_anchor_uses_latest_contract_row_not_parquet_filename_end() { + let dir = TestDir::new("chart_anchor_actual_contract_block"); + // File name claims coverage up to block 500 but the only row is at 300. + write_contract_parquet( + &dir.path + .join("contracts__chain_0000000100__0000000300__0000000500.parquet"), + 300, + 0x13, + 100, + ); + + let db = Db::open_with_mode(&dir.path, "*.parquet", false).unwrap(); + + assert_eq!(db.highest_block(100).await.unwrap(), Some(500)); + assert_eq!(db.highest_contract_block(100).await.unwrap(), Some(300)); +} + +#[tokio::test] +async fn stats_and_recent_filter_gnosis_chain() { + let dir = TestDir::new("gnosis_chain_filter"); + insert_zellic_snapshot(&dir.path); + write_backfill_parquet(&dir.path); + + let db = Db::open_with_mode(&dir.path, "*.parquet", false).unwrap(); + + let stats = db.stats(100).await.unwrap(); + assert_eq!(stats.total_contracts, 1); + assert_eq!(stats.first_block, 300); + assert_eq!(stats.last_block, 300); + + let recent = db.recent_contracts(100, 5, None).await.unwrap(); + assert_eq!(recent.contracts.len(), 1); + assert_eq!(recent.contracts[0].block_number, 300); + assert_eq!( + recent.contracts[0].address, + format!("0x{}", hex_string(0x15, 20)) + ); +} + +#[tokio::test] +async fn recent_does_not_fall_back_to_ethereum_for_other_chains() { + let dir = TestDir::new("no_cross_chain_recent_fallback"); + insert_zellic_snapshot(&dir.path); + + let db = Db::open_with_mode(&dir.path, "*.parquet", false).unwrap(); + + let recent = db.recent_contracts(100, 5, None).await.unwrap(); + assert!(recent.contracts.is_empty()); + assert!(!recent.has_more); +} + +#[tokio::test] +async fn dashboard_sql_scopes_contract_metadata_by_chain() { + let dir = TestDir::new("query_chain_scope"); + insert_zellic_snapshot(&dir.path); + write_backfill_parquet(&dir.path); + + let db = Db::open_with_mode(&dir.path, "*.parquet", false).unwrap(); + + let result = db + .query_sql( + "SELECT chain_id, block_number FROM contract_metadata ORDER BY block_number" + .to_string(), + 10, + Some(100), + ) + .await + .unwrap(); + + assert_eq!(result.row_count, 1); + assert_eq!(result.rows[0][0], serde_json::json!(100)); + assert_eq!(result.rows[0][1], serde_json::json!(300)); +} + +fn hex_string(byte: u8, len: usize) -> String { + hex::encode(vec![byte; len]) +} + +/// The SQL explorer's `contract_metadata` view must return identical data +/// before materialization (live-join fallback), after the background build, +/// and for fresh blocks beyond the materialized bounds (live union) — and a +/// backfill below the bounds must trigger a rebuild. +#[tokio::test] +async fn explorer_materialization_stays_correct_and_fresh() { + let dir = TestDir::new("explorer_materialization"); + write_backfill_parquet(&dir.path); + + let db = Db::open_with_mode(&dir.path, "*.parquet", false).unwrap(); + let explorer_query = "SELECT chain_id, block_number, address, compiler_version, is_verified \ + FROM contract_metadata ORDER BY block_number DESC, create_index DESC LIMIT 10"; + + // Fallback live-join view, pre-materialization. + let before = db + .query_sql(explorer_query.to_string(), 10, Some(1)) + .await + .unwrap(); + assert_eq!(before.row_count, 1); + assert_eq!(before.rows[0][1], serde_json::json!(200)); + + // Build the materialized table; a second refresh is a no-op. + assert!(db.refresh_explorer().await.unwrap()); + assert!(!db.refresh_explorer().await.unwrap()); + let after = db + .query_sql(explorer_query.to_string(), 10, Some(1)) + .await + .unwrap(); + assert_eq!(after.rows, before.rows); + + // New tail blocks beyond the bounds appear live without a rebuild. + write_contract_parquet( + &dir.path + .join("tail__chain_0000000001__0000000400__0000000400.parquet"), + 400, + 0x33, + 1, + ); + db.refresh().await.unwrap(); + let live = db + .query_sql(explorer_query.to_string(), 10, Some(1)) + .await + .unwrap(); + assert_eq!(live.row_count, 2); + assert_eq!(live.rows[0][1], serde_json::json!(400)); + assert!(!db.refresh_explorer().await.unwrap()); + + // A backfill below the materialized head forces a rebuild and the row + // shows up decorated. + write_contract_parquet( + &dir.path + .join("contracts__chain_0000000001__0000000150__0000000150.parquet"), + 150, + 0x43, + 1, + ); + db.refresh().await.unwrap(); + assert!(db.refresh_explorer().await.unwrap()); + let rebuilt = db + .query_sql(explorer_query.to_string(), 10, Some(1)) + .await + .unwrap(); + assert_eq!(rebuilt.row_count, 3); + assert_eq!( + rebuilt + .rows + .iter() + .map(|row| row[1].clone()) + .collect::>(), + vec![ + serde_json::json!(400), + serde_json::json!(200), + serde_json::json!(150) + ] + ); +} + +/// While a schema-upgrading explorer rebuild runs (or after a failed one), +/// the on-disk table is the previous generation without newer columns — +/// aggregates must fall back to the join path instead of binder-erroring. +#[tokio::test] +async fn aggregates_fall_back_when_explorer_table_has_old_schema() { + let dir = TestDir::new("explorer_old_schema_fallback"); + write_backfill_parquet(&dir.path); + + let db = Db::open_with_mode(&dir.path, "*.parquet", false).unwrap(); + // Simulate a previous-generation materialized table: right names, no + // is_decoded column. + db.execute_batch( + r#" + CREATE TABLE contract_metadata_native ( + chain_id UBIGINT, block_number UINTEGER, is_erc20 BOOLEAN + ); + CREATE TABLE IF NOT EXISTS contract_metadata_bounds ( + chain_id UBIGINT PRIMARY KEY, max_block UBIGINT NOT NULL + ); + "# + .to_string(), + ) + .await + .unwrap(); + + let sizes = db + .bytecode_size_distribution(ETHEREUM_CHAIN_ID, Some((0, 1000))) + .await + .unwrap(); + assert_eq!(sizes.iter().map(|bin| bin.count).sum::(), 1); + let standards = db + .standards_breakdown(ETHEREUM_CHAIN_ID, Some((0, 1000))) + .await + .unwrap(); + assert_eq!(standards.total_decoded, 0); + assert!(db.language_distribution(ETHEREUM_CHAIN_ID).await.is_ok()); +} + +/// The Zellic snapshot is ingested in ~1M-row block-range slices (one +/// transaction each) so it cannot OOM small hosts. 2.5M rows forces three +/// slices; totals must come out exact. +#[tokio::test] +async fn large_zellic_snapshot_ingests_in_slices() { + let dir = TestDir::new("zellic_sliced_ingest"); + { + let conn = Connection::open(dir.path.join("blink.duckdb")).unwrap(); + conn.execute_batch( + r#" + CREATE TABLE zellic_bytecodes ( + code_hash BLOB, + code BLOB, + n_code_bytes UINTEGER + ); + CREATE TABLE zellic_contracts AS + SELECT + substr(md5(('a' || i)::VARCHAR), 1, 20)::BLOB AS contract_address, + md5(('h' || i % 1000)::VARCHAR)::BLOB AS bytecode_hash, + (i % 500000)::UINTEGER AS block_number, + (i // 500000)::UINTEGER AS create_index, + 1::UBIGINT AS chain_id + FROM range(2500000) t(i); + INSERT INTO zellic_bytecodes + SELECT DISTINCT bytecode_hash, unhex('60'), 1::UINTEGER FROM zellic_contracts; + "#, + ) + .unwrap(); + } + + // A tight memory limit mirrors the production 4GB host where a + // single-transaction zellic ingest OOMed; sliced ingest must fit. + let db = Db::open( + &dir.path, + "*.parquet", + blink::db::DbOptions { + memory_limit: Some("500MB".to_string()), + ..Default::default() + }, + ) + .unwrap(); + let stats = db.stats(ETHEREUM_CHAIN_ID).await.unwrap(); + assert_eq!(stats.total_contracts, 2_500_000); + assert_eq!(stats.first_block, 0); + assert_eq!(stats.last_block, 499_999); + + // Reopening must not re-ingest (source recorded once after all slices). + drop(db); + let db = Db::open_with_mode(&dir.path, "*.parquet", false).unwrap(); + assert_eq!( + db.stats(ETHEREUM_CHAIN_ID).await.unwrap().total_contracts, + 2_500_000 + ); +} + +/// Ranged code aggregates read fully-covered 10k-block buckets from the +/// bucketed rollup and only scan the deployments table for partial edges — +/// results must be exact across all bucket-alignment cases. +#[tokio::test] +async fn ranged_code_aggregates_are_exact_across_bucket_boundaries() { + let dir = TestDir::new("bucket_boundary_ranges"); + // Lead edge (5000), one full bucket (10000..=19999, 10 rows), trail edge (25000). + let mut blocks: Vec = vec![5_000]; + blocks.extend((0..10).map(|i| 10_000 + i * 1_000)); + blocks.push(25_000); + write_multi_block_parquet( + &dir.path + .join("contracts__chain_0000000001__0000000000__0000030000.parquet"), + &blocks, + 1, + ); + + let db = Db::open_with_mode(&dir.path, "*.parquet", false).unwrap(); + db.execute_batch( + r#" + INSERT INTO bytecode_metadata_by_hash ( + code_hash, language, compiler_version, has_source_hash, + is_erc20, is_erc721, is_erc1155, is_proxy_eip1967, + is_proxy_minimal, uses_push0 + ) VALUES (unhex(repeat('aa', 32)), 'solidity', '0.8.24', true, false, false, false, false, false, true) + "# + .to_string(), + ) + .await + .unwrap(); + + // Edges + full interior bucket. + assert_eq!( + db.compiler_version_total(1, Some((5_000, 25_000))) + .await + .unwrap(), + 12 + ); + // Exactly one full bucket, aligned on both sides. + assert_eq!( + db.compiler_version_total(1, Some((10_000, 19_999))) + .await + .unwrap(), + 10 + ); + // No fully covered bucket: pure edge scan. + assert_eq!( + db.compiler_version_total(1, Some((5_000, 9_999))) + .await + .unwrap(), + 1 + ); + // Range wider than the data. + assert_eq!( + db.compiler_version_total(1, Some((0, 1_000_000))) + .await + .unwrap(), + 12 + ); + // Standards and sizes flow through the same source. + let standards = db + .standards_breakdown(1, Some((5_000, 25_000))) + .await + .unwrap(); + assert_eq!(standards.uses_push0, 12); + assert_eq!(standards.total_decoded, 12); + let sizes = db + .bytecode_size_distribution(1, Some((5_000, 25_000))) + .await + .unwrap(); + assert_eq!(sizes.iter().map(|bin| bin.count).sum::(), 12); + + // Once the materialized explorer table exists, the same endpoints switch + // to denormalized per-deployment scans — results must be identical. + assert!(db.refresh_explorer().await.unwrap()); + assert_eq!( + db.compiler_version_total(1, Some((5_000, 25_000))) + .await + .unwrap(), + 12 + ); + assert_eq!( + db.compiler_version_total(1, Some((10_000, 19_999))) + .await + .unwrap(), + 10 + ); + assert_eq!( + db.compiler_version_total(1, Some((5_000, 9_999))) + .await + .unwrap(), + 1 + ); + let compilers = db + .top_compilers(1, 12, Some((5_000, 25_000))) + .await + .unwrap(); + assert_eq!(compilers.len(), 1); + assert_eq!(compilers[0].count, 12); + let standards = db + .standards_breakdown(1, Some((5_000, 25_000))) + .await + .unwrap(); + assert_eq!(standards.uses_push0, 12); + assert_eq!(standards.has_source_hash, 12); + assert_eq!(standards.total_decoded, 12); + let sizes = db + .bytecode_size_distribution(1, Some((5_000, 25_000))) + .await + .unwrap(); + assert_eq!(sizes.iter().map(|bin| bin.count).sum::(), 12); + let languages = db.language_distribution(1).await.unwrap(); + assert_eq!(languages.len(), 1); + assert_eq!(languages[0].language, "solidity"); + assert_eq!(languages[0].count, 12); +} + +/// `blink load --overwrite` re-imports the Zellic snapshot; the invalidation +/// hook must subtract the old rollup contribution so the fresh snapshot is +/// re-ingested on the next open. +#[tokio::test] +async fn zellic_overwrite_invalidation_reingests_fresh_snapshot() { + let dir = TestDir::new("zellic_overwrite_invalidation"); + insert_zellic_snapshot(&dir.path); + { + let db = Db::open_with_mode(&dir.path, "*.parquet", false).unwrap(); + assert_eq!( + db.stats(ETHEREUM_CHAIN_ID).await.unwrap().total_contracts, + 1 + ); + } + + // Simulate the `load --overwrite` flow: invalidate, then rebuild the + // snapshot tables with different contents (two contracts now). + { + let conn = Connection::open(dir.path.join("blink.duckdb")).unwrap(); + blink::db::invalidate_zellic_rollups(&conn).unwrap(); + conn.execute_batch( + r#" + DROP TABLE IF EXISTS zellic_contracts; + DROP TABLE IF EXISTS zellic_bytecodes; + CREATE TABLE zellic_bytecodes ( + code_hash BLOB, + code BLOB, + n_code_bytes UINTEGER + ); + CREATE TABLE zellic_contracts ( + contract_address BLOB, + bytecode_hash BLOB, + block_number UINTEGER, + create_index UINTEGER, + chain_id UBIGINT + ); + INSERT INTO zellic_bytecodes VALUES (unhex(repeat('01', 32)), unhex('60606060'), 4); + INSERT INTO zellic_contracts VALUES (unhex(repeat('02', 20)), unhex(repeat('01', 32)), 100, 0, 1); + INSERT INTO zellic_contracts VALUES (unhex(repeat('04', 20)), unhex(repeat('01', 32)), 150, 0, 1); + "#, + ) + .unwrap(); + } + + let db = Db::open_with_mode(&dir.path, "*.parquet", false).unwrap(); + let stats = db.stats(ETHEREUM_CHAIN_ID).await.unwrap(); + assert_eq!(stats.total_contracts, 2); + assert_eq!(stats.first_block, 100); + assert_eq!(stats.last_block, 150); + + let deploys = db + .deploys_over_time(ETHEREUM_CHAIN_ID, 50, Some((0, 200))) + .await + .unwrap(); + assert_eq!( + deploys + .iter() + .map(|bucket| (bucket.block_start, bucket.count)) + .collect::>(), + vec![(100, 1), (150, 1)] + ); +} diff --git a/tests/decode.rs b/tests/decode.rs new file mode 100644 index 0000000..59a482a --- /dev/null +++ b/tests/decode.rs @@ -0,0 +1,84 @@ +//! Bytecode metadata decoding tests. + +use blink::decode::bytecode_meta::{analyze, Language}; + +#[test] +fn empty_bytecode() { + let m = analyze(&[]); + assert!(m.compiler_version.is_none()); + assert!(!m.is_erc20); +} + +#[test] +fn detects_push0() { + let code = [0x5f, 0x00]; + let m = analyze(&code); + assert!(m.uses_push0); +} + +#[test] +fn detects_erc20_selectors() { + let mut code = vec![]; + for sel in [ + [0x18u8, 0x16, 0x0d, 0xdd], + [0xa9, 0x05, 0x9c, 0xbb], + [0xdd, 0x62, 0xed, 0x3e], + ] { + code.push(0x63); + code.extend_from_slice(&sel); + } + let m = analyze(&code); + assert!(m.is_erc20); +} + +#[test] +fn parses_solc_metadata() { + // CBOR: a1 64 73 6f 6c 63 43 00 08 14 → { "solc": h'000814' } (= 0.8.20) + // length suffix: 00 0a (10 bytes) + let code = vec![ + 0xa1, 0x64, 0x73, 0x6f, 0x6c, 0x63, 0x43, 0x00, 0x08, 0x14, 0x00, 0x0a, + ]; + let m = analyze(&code); + assert_eq!(m.language, Some(Language::Solidity)); + assert_eq!(m.compiler_version.as_deref(), Some("0.8.20")); +} + +#[test] +fn detects_eip1167_minimal_proxy() { + // Construct the canonical 45-byte runtime with a dummy impl address. + let mut code = vec![]; + code.extend_from_slice(&[0x36, 0x3d, 0x3d, 0x37, 0x3d, 0x3d, 0x3d, 0x36, 0x3d, 0x73]); + code.extend_from_slice(&[0xab; 20]); // implementation address + code.extend_from_slice(&[ + 0x5a, 0xf4, 0x3d, 0x82, 0x80, 0x3e, 0x90, 0x3d, 0x91, 0x60, 0x2b, 0x57, 0xfd, 0x5b, 0xf3, + ]); + assert_eq!(code.len(), 45); + let m = analyze(&code); + assert!(m.is_proxy_minimal, "should detect EIP-1167 minimal proxy"); + // Should not also flag as a different proxy type. + assert!(!m.is_proxy_eip1967); +} + +#[test] +fn rejects_minimal_proxy_with_wrong_length() { + // Same shape but one byte short — not a valid EIP-1167. + let mut code = vec![0x36, 0x3d, 0x3d, 0x37, 0x3d, 0x3d, 0x3d, 0x36, 0x3d, 0x73]; + code.extend_from_slice(&[0xab; 19]); + code.extend_from_slice(&[ + 0x5a, 0xf4, 0x3d, 0x82, 0x80, 0x3e, 0x90, 0x3d, 0x91, 0x60, 0x2b, 0x57, 0xfd, 0x5b, 0xf3, + ]); + assert_eq!(code.len(), 44); + let m = analyze(&code); + assert!(!m.is_proxy_minimal); +} + +#[test] +fn ignores_absurd_cbor_map_count() { + let code = vec![ + 0xba, 0x99, 0xb6, 0x26, 0x57, // map(2578851415) + 0x00, 0x05, // metadata length: 5 bytes + ]; + let m = analyze(&code); + assert!(m.language.is_none()); + assert!(m.compiler_version.is_none()); +} diff --git a/tests/extract_batch.rs b/tests/extract_batch.rs new file mode 100644 index 0000000..f0afafa --- /dev/null +++ b/tests/extract_batch.rs @@ -0,0 +1,27 @@ +//! Trace batch decoding tests. + +use blink::extract::batch::decode_trace_block_value; + +#[test] +fn trace_decode_ignores_gnosis_external_reward_traces() { + let value = serde_json::json!([ + { + "action": { + "author": "0x0000000000000000000000000000000000000000", + "rewardType": "external", + "value": "0x0" + }, + "blockHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "blockNumber": 46630628, + "result": null, + "subtraces": 0, + "traceAddress": [], + "transactionHash": null, + "transactionPosition": null, + "type": "reward" + } + ]); + + let traces = decode_trace_block_value(46630628, value).unwrap(); + assert!(traces.is_empty()); +} diff --git a/tests/load_inputs.rs b/tests/load_inputs.rs new file mode 100644 index 0000000..dcc20ed --- /dev/null +++ b/tests/load_inputs.rs @@ -0,0 +1,147 @@ +//! Input detection tests for `blink load`. + +use std::{ + fs, + path::PathBuf, + time::{SystemTime, UNIX_EPOCH}, +}; + +use blink::load::{ + detect_inputs, detect_verifier_alliance_inputs, list_parquet_files, load_parquet_links, +}; + +struct TestDir { + path: PathBuf, +} + +impl TestDir { + fn new(name: &str) -> Self { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let path = std::env::temp_dir().join(format!( + "blink_load_test_{}_{}_{}", + std::process::id(), + name, + unique + )); + fs::create_dir_all(&path).unwrap(); + Self { path } + } + + fn touch(&self, name: &str) -> PathBuf { + let path = self.path.join(name); + fs::write(&path, []).unwrap(); + path + } +} + +impl Drop for TestDir { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.path); + } +} + +fn names(paths: &[PathBuf]) -> Vec { + paths + .iter() + .map(|path| { + path.file_name() + .and_then(|name| name.to_str()) + .unwrap() + .to_string() + }) + .collect() +} + +#[test] +fn detect_inputs_requires_both_csv_files_for_csv_load() { + let dir = TestDir::new("csv_pair"); + dir.touch("contracts.csv"); + + let inputs = detect_inputs(&dir.path, "*.parquet").unwrap(); + + assert!(!inputs.has_normalized_csv); + assert!(inputs.parquet_files.is_empty()); + + dir.touch("bytecodes.csv"); + let inputs = detect_inputs(&dir.path, "*.parquet").unwrap(); + + assert!(inputs.has_normalized_csv); + assert_eq!(inputs.csv_contracts, dir.path.join("contracts.csv")); + assert_eq!(inputs.csv_bytecodes, dir.path.join("bytecodes.csv")); +} + +#[test] +fn list_parquet_files_filters_hidden_and_non_matching_files() { + let dir = TestDir::new("parquet_filter"); + dir.touch("b.parquet"); + dir.touch("a.parquet"); + dir.touch(".hidden.parquet"); + dir.touch("contracts.csv"); + dir.touch("notes.txt"); + + let files = list_parquet_files(&dir.path, "*.parquet").unwrap(); + + assert_eq!(names(&files), vec!["a.parquet", "b.parquet"]); +} + +#[test] +fn detect_inputs_applies_parquet_glob() { + let dir = TestDir::new("parquet_glob"); + dir.touch("ethereum__contracts__1_to_2.parquet"); + dir.touch("other__contracts__1_to_2.parquet"); + + let inputs = detect_inputs(&dir.path, "ethereum__*.parquet").unwrap(); + + assert_eq!( + names(&inputs.parquet_files), + vec!["ethereum__contracts__1_to_2.parquet"] + ); +} + +#[test] +fn detect_verifier_alliance_inputs_requires_both_tables() { + let dir = TestDir::new("va_missing_table"); + fs::create_dir_all(dir.path.join("contract_deployments")).unwrap(); + + let err = detect_verifier_alliance_inputs(Some(&dir.path)).unwrap_err(); + + assert!(err.to_string().contains("--va needs both")); +} + +#[test] +fn detect_verifier_alliance_inputs_finds_required_parquet_files() { + let dir = TestDir::new("va_tables"); + let deployments = dir.path.join("contract_deployments"); + let verifications = dir.path.join("verified_contracts"); + fs::create_dir_all(&deployments).unwrap(); + fs::create_dir_all(&verifications).unwrap(); + fs::write(deployments.join("contract_deployments_0_1.parquet"), []).unwrap(); + fs::write(verifications.join("verified_contracts_0_1.parquet"), []).unwrap(); + + let inputs = detect_verifier_alliance_inputs(Some(&dir.path)) + .unwrap() + .unwrap(); + + assert_eq!(inputs.contract_deployments.len(), 1); + assert_eq!(inputs.verified_contracts.len(), 1); +} + +#[test] +fn load_parquet_links_creates_symlinks_without_copying() { + let src = TestDir::new("link_src"); + let dst = TestDir::new("link_dst"); + let parquet = src.touch("contracts__0000000001__0000000002.parquet"); + + load_parquet_links(&src.path, &dst.path, std::slice::from_ref(&parquet), false).unwrap(); + + let linked = dst.path.join("contracts__0000000001__0000000002.parquet"); + let metadata = fs::symlink_metadata(&linked).unwrap(); + assert!(metadata.file_type().is_symlink()); + assert_eq!( + fs::canonicalize(linked).unwrap(), + fs::canonicalize(parquet).unwrap() + ); +} diff --git a/tests/serve_windows.rs b/tests/serve_windows.rs new file mode 100644 index 0000000..388b094 --- /dev/null +++ b/tests/serve_windows.rs @@ -0,0 +1,190 @@ +//! Time-window parsing, cache-key normalization, and runtime-state tests for +//! the dashboard server. + +use blink::chains::{ETHEREUM_CHAIN_ID, GNOSIS_CHAIN_ID}; +use blink::serve::{ + bucket_cache_key, parse_time_series_window, range_cache_key_for_query, BucketQuery, + RuntimeState, +}; + +fn query(range: Option<&str>, bucket: Option<&str>) -> BucketQuery { + BucketQuery { + chain_id: None, + bucket: bucket.map(str::to_string), + range: range.map(str::to_string), + end_block: None, + start_block: None, + start_time: None, + end_time: None, + limit: None, + } +} + +#[test] +fn day_range_limits_chart_to_last_day_with_hourly_buckets() { + let anchor_block = 20_000_000; + let window = + parse_time_series_window(&query(Some("day"), None), ETHEREUM_CHAIN_ID, anchor_block); + + assert_eq!(window.block_range, Some((19_992_801, 20_000_000))); + assert_eq!(window.bucket_blocks, 300); +} + +#[test] +fn week_range_limits_chart_to_last_week_with_daily_buckets() { + let anchor_block = 20_000_000; + let window = + parse_time_series_window(&query(Some("week"), None), ETHEREUM_CHAIN_ID, anchor_block); + + assert_eq!(window.block_range, Some((19_949_601, 20_000_000))); + assert_eq!(window.bucket_blocks, 7_200); +} + +#[test] +fn year_range_limits_chart_to_last_year_with_monthly_buckets() { + let anchor_block = 20_000_000; + let window = + parse_time_series_window(&query(Some("year"), None), ETHEREUM_CHAIN_ID, anchor_block); + + assert_eq!(window.block_range, Some((17_372_001, 20_000_000))); + assert_eq!(window.bucket_blocks, 216_000); +} + +#[test] +fn hour_range_uses_chain_specific_block_time() { + let anchor_block = 46_000_000; + let window = + parse_time_series_window(&query(Some("hour"), None), GNOSIS_CHAIN_ID, anchor_block); + + assert_eq!(window.block_range, Some((45_999_281, 46_000_000))); + assert_eq!(window.bucket_blocks, 60); +} + +#[test] +fn legacy_bucket_query_keeps_full_history_behavior() { + let anchor_block = 20_000_000; + let window = + parse_time_series_window(&query(None, Some("day")), ETHEREUM_CHAIN_ID, anchor_block); + + assert_eq!(window.block_range, None); + assert_eq!(window.bucket_blocks, 7_200); +} + +#[test] +fn range_end_block_moves_visible_window() { + let anchor_block = 20_000_000; + let mut q = query(Some("day"), None); + q.end_block = Some(19_000_000); + let window = parse_time_series_window(&q, ETHEREUM_CHAIN_ID, anchor_block); + + assert_eq!(window.block_range, Some((18_992_801, 19_000_000))); + + q.end_block = Some(21_000_000); + let capped = parse_time_series_window(&q, ETHEREUM_CHAIN_ID, anchor_block); + assert_eq!(capped.block_range, Some((19_992_801, 20_000_000))); +} + +#[test] +fn relative_preset_cache_key_survives_tail_moves_inside_bucket() { + let q = query(Some("day"), None); + let first = parse_time_series_window(&q, ETHEREUM_CHAIN_ID, 20_000_001); + let second = parse_time_series_window(&q, ETHEREUM_CHAIN_ID, 20_000_099); + + assert_ne!(first.block_range, second.block_range); + assert_eq!( + bucket_cache_key(ETHEREUM_CHAIN_ID, &q, first), + bucket_cache_key(ETHEREUM_CHAIN_ID, &q, second) + ); + assert_eq!( + range_cache_key_for_query(ETHEREUM_CHAIN_ID, &q, first), + range_cache_key_for_query(ETHEREUM_CHAIN_ID, &q, second) + ); +} + +#[test] +fn explicit_block_range_cache_key_keeps_exact_window() { + let anchor_block = 20_000_100; + let mut first_query = query(Some("day"), None); + first_query.end_block = Some(20_000_001); + let first = parse_time_series_window(&first_query, ETHEREUM_CHAIN_ID, anchor_block); + + let mut second_query = query(Some("day"), None); + second_query.end_block = Some(20_000_099); + let second = parse_time_series_window(&second_query, ETHEREUM_CHAIN_ID, anchor_block); + + assert_ne!(first.block_range, second.block_range); + assert_ne!( + bucket_cache_key(ETHEREUM_CHAIN_ID, &first_query, first), + bucket_cache_key(ETHEREUM_CHAIN_ID, &second_query, second) + ); +} + +#[test] +fn explicit_start_block_creates_custom_window() { + let anchor_block = 20_000_000; + let mut q = query(None, None); + q.start_block = Some(19_900_000); + q.end_block = Some(19_950_000); + let window = parse_time_series_window(&q, ETHEREUM_CHAIN_ID, anchor_block); + + assert_eq!(window.block_range, Some((19_900_000, 19_950_000))); + assert_eq!(window.bucket_blocks, 520); +} + +#[tokio::test] +async fn gnosis_tail_updates_do_not_overwrite_legacy_ethereum_runtime_block() { + let runtime = RuntimeState::new(false, true, 60); + + runtime + .mark_tail_ready(ETHEREUM_CHAIN_ID, Some(25_438_551)) + .await; + runtime + .mark_tail_ready(GNOSIS_CHAIN_ID, Some(46_981_003)) + .await; + + let response = runtime.response().await; + assert_eq!(response.snapshot.tail_last_block, Some(25_438_551)); + + let gnosis = response + .snapshot + .tail_chains + .iter() + .find(|chain| chain.chain_id == GNOSIS_CHAIN_ID) + .expect("gnosis runtime state"); + assert_eq!(gnosis.tail_last_block, Some(46_981_003)); + + runtime + .mark_tail_ok(Some(GNOSIS_CHAIN_ID), Some(46_981_010), 8) + .await; + let response = runtime.response().await; + assert_eq!(response.snapshot.tail_last_block, Some(25_438_551)); + + runtime + .mark_tail_ok(Some(ETHEREUM_CHAIN_ID), Some(25_438_552), 13) + .await; + let response = runtime.response().await; + assert_eq!(response.snapshot.tail_last_block, Some(25_438_552)); +} + +#[tokio::test] +async fn ready_chain_does_not_flicker_to_tailing_during_background_scan() { + let runtime = RuntimeState::new(false, true, 60); + + runtime + .mark_tail_ready(ETHEREUM_CHAIN_ID, Some(25_438_551)) + .await; + runtime.mark_tail_start(Some(ETHEREUM_CHAIN_ID)).await; + + let response = runtime.response().await; + assert!(!response.snapshot.tail_running); + assert_eq!(response.snapshot.tail_running_count, 0); + + let ethereum = response + .snapshot + .tail_chains + .iter() + .find(|chain| chain.chain_id == ETHEREUM_CHAIN_ID) + .expect("ethereum runtime state"); + assert!(!ethereum.tail_running); + assert_eq!(ethereum.tail_running_count, 0); +}